1
2
3
4 """
5 This file is part of the web2py Web Framework
6 Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
7 License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
8
9 Thanks to
10 * Niall Sweeny <niall.sweeny@fonjax.com> for MS SQL support
11 * Marcel Leuthi <mluethi@mlsystems.ch> for Oracle support
12 * Denes
13 * Chris Clark
14 * clach05
15 * Denes Lengyel
16 * and many others who have contributed to current and previous versions
17
18 This file contains the DAL support for many relational databases,
19 including:
20 - SQLite & SpatiaLite
21 - MySQL
22 - Postgres
23 - Firebird
24 - Oracle
25 - MS SQL
26 - DB2
27 - Interbase
28 - Ingres
29 - Informix (9+ and SE)
30 - SapDB (experimental)
31 - Cubrid (experimental)
32 - CouchDB (experimental)
33 - MongoDB (in progress)
34 - Google:nosql
35 - Google:sql
36 - Teradata
37 - IMAP (experimental)
38
39 Example of usage:
40
41 >>> # from dal import DAL, Field
42
43 ### create DAL connection (and create DB if it doesn't exist)
44 >>> db = DAL(('sqlite://storage.sqlite','mysql://a:b@localhost/x'),
45 ... folder=None)
46
47 ### define a table 'person' (create/alter as necessary)
48 >>> person = db.define_table('person',Field('name','string'))
49
50 ### insert a record
51 >>> id = person.insert(name='James')
52
53 ### retrieve it by id
54 >>> james = person(id)
55
56 ### retrieve it by name
57 >>> james = person(name='James')
58
59 ### retrieve it by arbitrary query
60 >>> query = (person.name=='James') & (person.name.startswith('J'))
61 >>> james = db(query).select(person.ALL)[0]
62
63 ### update one record
64 >>> james.update_record(name='Jim')
65 <Row {'id': 1, 'name': 'Jim'}>
66
67 ### update multiple records by query
68 >>> db(person.name.like('J%')).update(name='James')
69 1
70
71 ### delete records by query
72 >>> db(person.name.lower() == 'jim').delete()
73 0
74
75 ### retrieve multiple records (rows)
76 >>> people = db(person).select(orderby=person.name,
77 ... groupby=person.name, limitby=(0,100))
78
79 ### further filter them
80 >>> james = people.find(lambda row: row.name == 'James').first()
81 >>> print james.id, james.name
82 1 James
83
84 ### check aggregates
85 >>> counter = person.id.count()
86 >>> print db(person).select(counter).first()(counter)
87 1
88
89 ### delete one record
90 >>> james.delete_record()
91 1
92
93 ### delete (drop) entire database table
94 >>> person.drop()
95
96 Supported field types:
97 id string text boolean integer double decimal password upload
98 blob time date datetime
99
100 Supported DAL URI strings:
101 'sqlite://test.db'
102 'spatialite://test.db'
103 'sqlite:memory'
104 'spatialite:memory'
105 'jdbc:sqlite://test.db'
106 'mysql://root:none@localhost/test'
107 'postgres://mdipierro:password@localhost/test'
108 'postgres:psycopg2://mdipierro:password@localhost/test'
109 'postgres:pg8000://mdipierro:password@localhost/test'
110 'jdbc:postgres://mdipierro:none@localhost/test'
111 'mssql://web2py:none@A64X2/web2py_test'
112 'mssql2://web2py:none@A64X2/web2py_test' # alternate mappings
113 'oracle://username:password@database'
114 'firebird://user:password@server:3050/database'
115 'db2://DSN=dsn;UID=user;PWD=pass'
116 'firebird://username:password@hostname/database'
117 'firebird_embedded://username:password@c://path'
118 'informix://user:password@server:3050/database'
119 'informixu://user:password@server:3050/database' # unicode informix
120 'ingres://database' # or use an ODBC connection string, e.g. 'ingres://dsn=dsn_name'
121 'google:datastore' # for google app engine datastore
122 'google:sql' # for google app engine with sql (mysql compatible)
123 'teradata://DSN=dsn;UID=user;PWD=pass; DATABASE=database' # experimental
124 'imap://user:password@server:port' # experimental
125
126 For more info:
127 help(DAL)
128 help(Field)
129 """
130
131
132
133
134
135 __all__ = ['DAL', 'Field']
136
137 MAXCHARLENGTH = 2**15
138 DEFAULTLENGTH = {'string':512,
139 'password':512,
140 'upload':512,
141 'text':2**15,
142 'blob':2**31}
143 TIMINGSSIZE = 100
144 SPATIALLIBS = {
145 'Windows':'libspatialite',
146 'Linux':'libspatialite.so',
147 'Darwin':'libspatialite.dylib'
148 }
149 DEFAULT_URI = 'sqlite://dummy.db'
150
151 import re
152 import sys
153 import locale
154 import os
155 import types
156 import datetime
157 import threading
158 import time
159 import csv
160 import cgi
161 import copy
162 import socket
163 import logging
164 import base64
165 import shutil
166 import marshal
167 import decimal
168 import struct
169 import urllib
170 import hashlib
171 import uuid
172 import glob
173 import traceback
174 import platform
175
176 PYTHON_VERSION = sys.version_info[0]
177 if PYTHON_VERSION == 2:
178 import cPickle as pickle
179 import cStringIO as StringIO
180 import copy_reg as copyreg
181 hashlib_md5 = hashlib.md5
182 bytes, unicode = str, unicode
183 else:
184 import pickle
185 from io import StringIO as StringIO
186 import copyreg
187 long = int
188 hashlib_md5 = lambda s: hashlib.md5(bytes(s,'utf8'))
189 bytes, unicode = bytes, str
190
191 CALLABLETYPES = (types.LambdaType, types.FunctionType,
192 types.BuiltinFunctionType,
193 types.MethodType, types.BuiltinMethodType)
194
195 TABLE_ARGS = set(
196 ('migrate','primarykey','fake_migrate','format','redefine',
197 'singular','plural','trigger_name','sequence_name',
198 'common_filter','polymodel','table_class','on_define',))
199
200 SELECT_ARGS = set(
201 ('orderby', 'groupby', 'limitby','required', 'cache', 'left',
202 'distinct', 'having', 'join','for_update', 'processor','cacheable'))
203
204 ogetattr = object.__getattribute__
205 osetattr = object.__setattr__
206 exists = os.path.exists
207 pjoin = os.path.join
208
209
210
211
212 try:
213 from utils import web2py_uuid
214 except (ImportError, SystemError):
215 import uuid
217
218 try:
219 import portalocker
220 have_portalocker = True
221 except ImportError:
222 have_portalocker = False
223
224 try:
225 import serializers
226 have_serializers = True
227 except ImportError:
228 have_serializers = False
229 try:
230 import json as simplejson
231 except ImportError:
232 try:
233 import gluon.contrib.simplejson as simplejson
234 except ImportError:
235 simplejson = None
236
237 try:
238 import validators
239 have_validators = True
240 except (ImportError, SyntaxError):
241 have_validators = False
242
243 LOGGER = logging.getLogger("web2py.dal")
244 DEFAULT = lambda:0
245
246 GLOBAL_LOCKER = threading.RLock()
247 THREAD_LOCAL = threading.local()
248
249
250
251
252 REGEX_TYPE = re.compile('^([\w\_\:]+)')
253 REGEX_DBNAME = re.compile('^(\w+)(\:\w+)*')
254 REGEX_W = re.compile('^\w+$')
255 REGEX_TABLE_DOT_FIELD = re.compile('^(\w+)\.(\w+)$')
256 REGEX_UPLOAD_PATTERN = re.compile('(?P<table>[\w\-]+)\.(?P<field>[\w\-]+)\.(?P<uuidkey>[\w\-]+)\.(?P<name>\w+)\.\w+$')
257 REGEX_CLEANUP_FN = re.compile('[\'"\s;]+')
258 REGEX_UNPACK = re.compile('(?<!\|)\|(?!\|)')
259 REGEX_PYTHON_KEYWORDS = re.compile('^(and|del|from|not|while|as|elif|global|or|with|assert|else|if|pass|yield|break|except|import|print|class|exec|in|raise|continue|finally|is|return|def|for|lambda|try)$')
260 REGEX_SELECT_AS_PARSER = re.compile("\s+AS\s+(\S+)")
261 REGEX_CONST_STRING = re.compile('(\"[^\"]*?\")|(\'[^\']*?\')')
262 REGEX_SEARCH_PATTERN = re.compile('^{[^\.]+\.[^\.]+(\.(lt|gt|le|ge|eq|ne|contains|startswith|year|month|day|hour|minute|second))?(\.not)?}$')
263 REGEX_SQUARE_BRACKETS = re.compile('^.+\[.+\]$')
264 REGEX_STORE_PATTERN = re.compile('\.(?P<e>\w{1,5})$')
265 REGEX_QUOTES = re.compile("'[^']*'")
266 REGEX_ALPHANUMERIC = re.compile('^[0-9a-zA-Z]\w*$')
267 REGEX_PASSWORD = re.compile('\://([^:@]*)\:')
268 REGEX_NOPASSWD = re.compile('(?<=\:)([^:@/]+)(?=@.+)')
269
270
271
272 DRIVERS = []
273
274 try:
275 from new import classobj
276 from google.appengine.ext import db as gae
277 from google.appengine.api import namespace_manager, rdbms
278 from google.appengine.api.datastore_types import Key
279 from google.appengine.ext.db.polymodel import PolyModel
280 DRIVERS.append('google')
281 except ImportError:
282 pass
283
284 if not 'google' in DRIVERS:
285
286 try:
287 from pysqlite2 import dbapi2 as sqlite2
288 DRIVERS.append('SQLite(sqlite2)')
289 except ImportError:
290 LOGGER.debug('no SQLite drivers pysqlite2.dbapi2')
291
292 try:
293 from sqlite3 import dbapi2 as sqlite3
294 DRIVERS.append('SQLite(sqlite3)')
295 except ImportError:
296 LOGGER.debug('no SQLite drivers sqlite3')
297
298 try:
299
300 try:
301 import contrib.pymysql as pymysql
302
303
304 pymysql.ESCAPE_REGEX = re.compile("'")
305 pymysql.ESCAPE_MAP = {"'": "''"}
306
307 except ImportError:
308 import pymysql
309 DRIVERS.append('MySQL(pymysql)')
310 except ImportError:
311 LOGGER.debug('no MySQL driver pymysql')
312
313 try:
314 import MySQLdb
315 DRIVERS.append('MySQL(MySQLdb)')
316 except ImportError:
317 LOGGER.debug('no MySQL driver MySQLDB')
318
319
320 try:
321 import psycopg2
322 from psycopg2.extensions import adapt as psycopg2_adapt
323 DRIVERS.append('PostgreSQL(psycopg2)')
324 except ImportError:
325 LOGGER.debug('no PostgreSQL driver psycopg2')
326
327 try:
328
329 try:
330 import contrib.pg8000.dbapi as pg8000
331 except ImportError:
332 import pg8000.dbapi as pg8000
333 DRIVERS.append('PostgreSQL(pg8000)')
334 except ImportError:
335 LOGGER.debug('no PostgreSQL driver pg8000')
336
337 try:
338 import cx_Oracle
339 DRIVERS.append('Oracle(cx_Oracle)')
340 except ImportError:
341 LOGGER.debug('no Oracle driver cx_Oracle')
342
343 try:
344 try:
345 import pyodbc
346 except ImportError:
347 try:
348 import contrib.pypyodbc as pyodbc
349 except Exception, e:
350 raise ImportError(str(e))
351 DRIVERS.append('MSSQL(pyodbc)')
352 DRIVERS.append('DB2(pyodbc)')
353 DRIVERS.append('Teradata(pyodbc)')
354 DRIVERS.append('Ingres(pyodbc)')
355 except ImportError:
356 LOGGER.debug('no MSSQL/DB2/Teradata/Ingres driver pyodbc')
357
358 try:
359 import Sybase
360 DRIVERS.append('Sybase(Sybase)')
361 except ImportError:
362 LOGGER.debug('no Sybase driver')
363
364 try:
365 import kinterbasdb
366 DRIVERS.append('Interbase(kinterbasdb)')
367 DRIVERS.append('Firebird(kinterbasdb)')
368 except ImportError:
369 LOGGER.debug('no Firebird/Interbase driver kinterbasdb')
370
371 try:
372 import fdb
373 DRIVERS.append('Firebird(fdb)')
374 except ImportError:
375 LOGGER.debug('no Firebird driver fdb')
376
377 try:
378 import firebirdsql
379 DRIVERS.append('Firebird(firebirdsql)')
380 except ImportError:
381 LOGGER.debug('no Firebird driver firebirdsql')
382
383 try:
384 import informixdb
385 DRIVERS.append('Informix(informixdb)')
386 LOGGER.warning('Informix support is experimental')
387 except ImportError:
388 LOGGER.debug('no Informix driver informixdb')
389
390 try:
391 import sapdb
392 DRIVERS.append('SQL(sapdb)')
393 LOGGER.warning('SAPDB support is experimental')
394 except ImportError:
395 LOGGER.debug('no SAP driver sapdb')
396
397 try:
398 import cubriddb
399 DRIVERS.append('Cubrid(cubriddb)')
400 LOGGER.warning('Cubrid support is experimental')
401 except ImportError:
402 LOGGER.debug('no Cubrid driver cubriddb')
403
404 try:
405 from com.ziclix.python.sql import zxJDBC
406 import java.sql
407
408 from org.sqlite import JDBC
409 zxJDBC_sqlite = java.sql.DriverManager
410 DRIVERS.append('PostgreSQL(zxJDBC)')
411 DRIVERS.append('SQLite(zxJDBC)')
412 LOGGER.warning('zxJDBC support is experimental')
413 is_jdbc = True
414 except ImportError:
415 LOGGER.debug('no SQLite/PostgreSQL driver zxJDBC')
416 is_jdbc = False
417
418 try:
419 import couchdb
420 DRIVERS.append('CouchDB(couchdb)')
421 except ImportError:
422 LOGGER.debug('no Couchdb driver couchdb')
423
424 try:
425 import pymongo
426 DRIVERS.append('MongoDB(pymongo)')
427 except:
428 LOGGER.debug('no MongoDB driver pymongo')
429
430 try:
431 import imaplib
432 DRIVERS.append('IMAP(imaplib)')
433 except:
434 LOGGER.debug('no IMAP driver imaplib')
435
436 PLURALIZE_RULES = [
437 (re.compile('child$'), re.compile('child$'), 'children'),
438 (re.compile('oot$'), re.compile('oot$'), 'eet'),
439 (re.compile('ooth$'), re.compile('ooth$'), 'eeth'),
440 (re.compile('l[eo]af$'), re.compile('l([eo])af$'), 'l\\1aves'),
441 (re.compile('sis$'), re.compile('sis$'), 'ses'),
442 (re.compile('man$'), re.compile('man$'), 'men'),
443 (re.compile('ife$'), re.compile('ife$'), 'ives'),
444 (re.compile('eau$'), re.compile('eau$'), 'eaux'),
445 (re.compile('lf$'), re.compile('lf$'), 'lves'),
446 (re.compile('[sxz]$'), re.compile('$'), 'es'),
447 (re.compile('[^aeioudgkprt]h$'), re.compile('$'), 'es'),
448 (re.compile('(qu|[^aeiou])y$'), re.compile('y$'), 'ies'),
449 (re.compile('$'), re.compile('$'), 's'),
450 ]
457
460
463
466
468
471
472 if 'google' in DRIVERS:
473
474 is_jdbc = False
477 """
478 GAE decimal implementation
479 """
480 data_type = decimal.Decimal
481
482 - def __init__(self, precision, scale, **kwargs):
483 super(GAEDecimalProperty, self).__init__(self, **kwargs)
484 d = '1.'
485 for x in range(scale):
486 d += '0'
487 self.round = decimal.Decimal(d)
488
496
498 if value is None or value == '':
499 return None
500 else:
501 return decimal.Decimal(value).quantize(self.round)
502
504 value = super(GAEDecimalProperty, self).validate(value)
505 if value is None or isinstance(value, decimal.Decimal):
506 return value
507 elif isinstance(value, basestring):
508 return decimal.Decimal(value)
509 raise gae.BadValueError("Property %s must be a Decimal or string."\
510 % self.name)
511
517
518 POOLS = {}
519 check_active_connection = True
520
521 @staticmethod
524
525
526
527 - def close(self,action='commit',really=True):
544
545 @staticmethod
547 """ to close cleanly databases in a multithreaded environment """
548 dbs = getattr(THREAD_LOCAL,'db_instances',{}).items()
549 for db_uid, db_group in dbs:
550 for db in db_group:
551 if hasattr(db,'_adapter'):
552 db._adapter.close(action)
553 getattr(THREAD_LOCAL,'db_instances',{}).clear()
554 getattr(THREAD_LOCAL,'db_instances_zombie',{}).clear()
555 if callable(action):
556 action(None)
557 return
558
560 """ this actually does not make the folder. it has to be there """
561 self.folder = getattr(THREAD_LOCAL,'folder','')
562
563
564 if False and self.folder and not exists(self.folder):
565 os.mkdir(self.folder)
566
568 """hook for the after_connection parameter"""
569 if callable(self._after_connection):
570 self._after_connection(self)
571 self.after_connection()
572
574 """ this it is supposed to be overloaded by adapters"""
575 pass
576
578 """
579 this function defines: self.connection and self.cursor
580 (iff cursor is True)
581 if self.pool_size>0 it will try pull the connection from the pool
582 if the connection is not active (closed by db server) it will loop
583 if not self.pool_size or no active connections in pool makes a new one
584 """
585 if getattr(self,'connection',None) != None:
586 return
587 if f is None:
588 f = self.connector
589
590 if not self.pool_size:
591 self.connection = f()
592 self.cursor = cursor and self.connection.cursor()
593 else:
594 uri = self.uri
595 POOLS = ConnectionPool.POOLS
596 while True:
597 GLOBAL_LOCKER.acquire()
598 if not uri in POOLS:
599 POOLS[uri] = []
600 if POOLS[uri]:
601 self.connection = POOLS[uri].pop()
602 GLOBAL_LOCKER.release()
603 self.cursor = cursor and self.connection.cursor()
604 try:
605 if self.cursor and self.check_active_connection:
606 self.execute('SELECT 1;')
607 break
608 except:
609 pass
610 else:
611 GLOBAL_LOCKER.release()
612 self.connection = f()
613 self.cursor = cursor and self.connection.cursor()
614 break
615 self.after_connection_hook()
616
623 native_json = False
624 driver = None
625 driver_name = None
626 drivers = ()
627 connection = None
628 maxcharlength = MAXCHARLENGTH
629 commit_on_alter_table = False
630 support_distributed_transaction = False
631 uploads_in_blob = False
632 can_select_for_update = True
633
634 TRUE = 'T'
635 FALSE = 'F'
636 T_SEP = ' '
637 types = {
638 'boolean': 'CHAR(1)',
639 'string': 'CHAR(%(length)s)',
640 'text': 'TEXT',
641 'json': 'TEXT',
642 'password': 'CHAR(%(length)s)',
643 'blob': 'BLOB',
644 'upload': 'CHAR(%(length)s)',
645 'integer': 'INTEGER',
646 'bigint': 'INTEGER',
647 'float':'DOUBLE',
648 'double': 'DOUBLE',
649 'decimal': 'DOUBLE',
650 'date': 'DATE',
651 'time': 'TIME',
652 'datetime': 'TIMESTAMP',
653 'id': 'INTEGER PRIMARY KEY AUTOINCREMENT',
654 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
655 'list:integer': 'TEXT',
656 'list:string': 'TEXT',
657 'list:reference': 'TEXT',
658
659 'big-id': 'BIGINT PRIMARY KEY AUTOINCREMENT',
660 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
661 }
662
664 return table._id != None
665
667 return "'%s'" % obj.replace("'", "''")
668
670 if isinstance(obj,(int,float)):
671 return str(obj)
672 return self.adapt(str(obj))
673
675 return self.driver.IntegrityError
676
678 return self.driver.OperationalError
679
681 """
682 to be used ONLY for files that on GAE may not be on filesystem
683 """
684 return exists(filename)
685
686 - def file_open(self, filename, mode='rb', lock=True):
687 """
688 to be used ONLY for files that on GAE may not be on filesystem
689 """
690 if have_portalocker and lock:
691 fileobj = portalocker.LockedFile(filename,mode)
692 else:
693 fileobj = open(filename,mode)
694 return fileobj
695
697 """
698 to be used ONLY for files that on GAE may not be on filesystem
699 """
700 if fileobj:
701 fileobj.close()
702
705
707 if getattr(self,'driver',None) != None:
708 return
709 drivers_available = [driver for driver in self.drivers
710 if driver in globals()]
711 if uri:
712 items = uri.split('://',1)[0].split(':')
713 request_driver = items[1] if len(items)>1 else None
714 else:
715 request_driver = None
716 request_driver = request_driver or adapter_args.get('driver')
717 if request_driver:
718 if request_driver in drivers_available:
719 self.driver_name = request_driver
720 self.driver = globals().get(request_driver)
721 else:
722 raise RuntimeError("driver %s not available" % request_driver)
723 elif drivers_available:
724 self.driver_name = drivers_available[0]
725 self.driver = globals().get(self.driver_name)
726 else:
727 raise RuntimeError("no driver available %s" % str(self.drivers))
728
729
730 - def __init__(self, db,uri,pool_size=0, folder=None, db_codec='UTF-8',
731 credential_decoder=IDENTITY, driver_args={},
732 adapter_args={},do_connect=True, after_connection=None):
733 self.db = db
734 self.dbengine = "None"
735 self.uri = uri
736 self.pool_size = pool_size
737 self.folder = folder
738 self.db_codec = db_codec
739 self._after_connection = after_connection
740 class Dummy(object):
741 lastrowid = 1
742 def __getattr__(self, value):
743 return lambda *a, **b: []
744 self.connection = Dummy()
745 self.cursor = Dummy()
746
748 return '%s_sequence' % tablename
749
751 return '%s_sequence' % tablename
752
755
756 - def create_table(self, table,
757 migrate=True,
758 fake_migrate=False,
759 polymodel=None):
760 db = table._db
761 fields = []
762
763 postcreation_fields = []
764 sql_fields = {}
765 sql_fields_aux = {}
766 TFK = {}
767 tablename = table._tablename
768 sortable = 0
769 types = self.types
770 for field in table:
771 sortable += 1
772 field_name = field.name
773 field_type = field.type
774 if isinstance(field_type,SQLCustomType):
775 ftype = field_type.native or field_type.type
776 elif field_type.startswith('reference'):
777 referenced = field_type[10:].strip()
778 if referenced == '.':
779 referenced = tablename
780 constraint_name = self.constraint_name(tablename, field_name)
781 if not '.' in referenced \
782 and referenced != tablename \
783 and hasattr(table,'_primarykey'):
784 ftype = types['integer']
785 else:
786 if hasattr(table,'_primarykey'):
787 rtablename,rfieldname = referenced.split('.')
788 rtable = db[rtablename]
789 rfield = rtable[rfieldname]
790
791 if rfieldname in rtable._primarykey or \
792 rfield.unique:
793 ftype = types[rfield.type[:9]] % \
794 dict(length=rfield.length)
795
796 if not rfield.unique and len(rtable._primarykey)>1:
797
798 if rtablename not in TFK:
799 TFK[rtablename] = {}
800 TFK[rtablename][rfieldname] = field_name
801 else:
802 ftype = ftype + \
803 types['reference FK'] % dict(
804 constraint_name = constraint_name,
805 foreign_key = '%s (%s)' % (rtablename,
806 rfieldname),
807 table_name = tablename,
808 field_name = field_name,
809 on_delete_action=field.ondelete)
810 else:
811
812 if referenced in db:
813 id_fieldname = db[referenced]._id.name
814 elif referenced == tablename:
815 id_fieldname = table._id.name
816 else:
817 id_fieldname = 'id'
818 ftype = types[field_type[:9]] % dict(
819 index_name = field_name+'__idx',
820 field_name = field_name,
821 constraint_name = constraint_name,
822 foreign_key = '%s (%s)' % (referenced,
823 id_fieldname),
824 on_delete_action=field.ondelete)
825 elif field_type.startswith('list:reference'):
826 ftype = types[field_type[:14]]
827 elif field_type.startswith('decimal'):
828 precision, scale = map(int,field_type[8:-1].split(','))
829 ftype = types[field_type[:7]] % \
830 dict(precision=precision,scale=scale)
831 elif field_type.startswith('geo'):
832 if not hasattr(self,'srid'):
833 raise RuntimeError('Adapter does not support geometry')
834 srid = self.srid
835 geotype, parms = field_type[:-1].split('(')
836 if not geotype in types:
837 raise SyntaxError(
838 'Field: unknown field type: %s for %s' \
839 % (field_type, field_name))
840 ftype = types[geotype]
841 if self.dbengine == 'postgres' and geotype == 'geometry':
842
843 dimension = 2
844 parms = parms.split(',')
845 if len(parms) == 3:
846 schema, srid, dimension = parms
847 elif len(parms) == 2:
848 schema, srid = parms
849 else:
850 schema = parms[0]
851 ftype = "SELECT AddGeometryColumn ('%%(schema)s', '%%(tablename)s', '%%(fieldname)s', %%(srid)s, '%s', %%(dimension)s);" % types[geotype]
852 ftype = ftype % dict(schema=schema,
853 tablename=tablename,
854 fieldname=field_name, srid=srid,
855 dimension=dimension)
856 postcreation_fields.append(ftype)
857 elif not field_type in types:
858 raise SyntaxError('Field: unknown field type: %s for %s' % \
859 (field_type, field_name))
860 else:
861 ftype = types[field_type]\
862 % dict(length=field.length)
863 if not field_type.startswith('id') and \
864 not field_type.startswith('reference'):
865 if field.notnull:
866 ftype += ' NOT NULL'
867 else:
868 ftype += self.ALLOW_NULL()
869 if field.unique:
870 ftype += ' UNIQUE'
871 if field.custom_qualifier:
872 ftype += ' %s' % field.custom_qualifier
873
874
875 sql_fields[field_name] = dict(
876 length=field.length,
877 unique=field.unique,
878 notnull=field.notnull,
879 sortable=sortable,
880 type=str(field_type),
881 sql=ftype)
882
883 if field.notnull and not field.default is None:
884
885
886
887
888
889
890 not_null = self.NOT_NULL(field.default, field_type)
891 ftype = ftype.replace('NOT NULL', not_null)
892 sql_fields_aux[field_name] = dict(sql=ftype)
893
894
895 if not (self.dbengine == 'postgres' and \
896 field_type.startswith('geom')):
897 fields.append('%s %s' % (field_name, ftype))
898 other = ';'
899
900
901 if self.dbengine == 'mysql':
902 if not hasattr(table, "_primarykey"):
903 fields.append('PRIMARY KEY(%s)' % table._id.name)
904 other = ' ENGINE=InnoDB CHARACTER SET utf8;'
905
906 fields = ',\n '.join(fields)
907 for rtablename in TFK:
908 rfields = TFK[rtablename]
909 pkeys = db[rtablename]._primarykey
910 fkeys = [ rfields[k] for k in pkeys ]
911 fields = fields + ',\n ' + \
912 types['reference TFK'] % dict(
913 table_name = tablename,
914 field_name=', '.join(fkeys),
915 foreign_table = rtablename,
916 foreign_key = ', '.join(pkeys),
917 on_delete_action = field.ondelete)
918
919 if hasattr(table,'_primarykey'):
920 query = "CREATE TABLE %s(\n %s,\n %s) %s" % \
921 (tablename, fields,
922 self.PRIMARY_KEY(', '.join(table._primarykey)),other)
923 else:
924 query = "CREATE TABLE %s(\n %s\n)%s" % \
925 (tablename, fields, other)
926
927 if self.uri.startswith('sqlite:///') \
928 or self.uri.startswith('spatialite:///'):
929 path_encoding = sys.getfilesystemencoding() \
930 or locale.getdefaultlocale()[1] or 'utf8'
931 dbpath = self.uri[9:self.uri.rfind('/')]\
932 .decode('utf8').encode(path_encoding)
933 else:
934 dbpath = self.folder
935
936 if not migrate:
937 return query
938 elif self.uri.startswith('sqlite:memory')\
939 or self.uri.startswith('spatialite:memory'):
940 table._dbt = None
941 elif isinstance(migrate, str):
942 table._dbt = pjoin(dbpath, migrate)
943 else:
944 table._dbt = pjoin(
945 dbpath, '%s_%s.table' % (table._db._uri_hash, tablename))
946
947 if table._dbt:
948 table._loggername = pjoin(dbpath, 'sql.log')
949 logfile = self.file_open(table._loggername, 'a')
950 else:
951 logfile = None
952 if not table._dbt or not self.file_exists(table._dbt):
953 if table._dbt:
954 logfile.write('timestamp: %s\n'
955 % datetime.datetime.today().isoformat())
956 logfile.write(query + '\n')
957 if not fake_migrate:
958 self.create_sequence_and_triggers(query,table)
959 table._db.commit()
960
961
962 for query in postcreation_fields:
963 self.execute(query)
964 table._db.commit()
965 if table._dbt:
966 tfile = self.file_open(table._dbt, 'w')
967 pickle.dump(sql_fields, tfile)
968 self.file_close(tfile)
969 if fake_migrate:
970 logfile.write('faked!\n')
971 else:
972 logfile.write('success!\n')
973 else:
974 tfile = self.file_open(table._dbt, 'r')
975 try:
976 sql_fields_old = pickle.load(tfile)
977 except EOFError:
978 self.file_close(tfile)
979 self.file_close(logfile)
980 raise RuntimeError('File %s appears corrupted' % table._dbt)
981 self.file_close(tfile)
982 if sql_fields != sql_fields_old:
983 self.migrate_table(table,
984 sql_fields, sql_fields_old,
985 sql_fields_aux, logfile,
986 fake_migrate=fake_migrate)
987 self.file_close(logfile)
988 return query
989
990 - def migrate_table(
991 self,
992 table,
993 sql_fields,
994 sql_fields_old,
995 sql_fields_aux,
996 logfile,
997 fake_migrate=False,
998 ):
999 db = table._db
1000 db._migrated.append(table._tablename)
1001 tablename = table._tablename
1002 def fix(item):
1003 k,v=item
1004 if not isinstance(v,dict):
1005 v=dict(type='unkown',sql=v)
1006 return k.lower(),v
1007
1008
1009 sql_fields = dict(map(fix,sql_fields.iteritems()))
1010 sql_fields_old = dict(map(fix,sql_fields_old.iteritems()))
1011 sql_fields_aux = dict(map(fix,sql_fields_aux.iteritems()))
1012 if db._debug:
1013 logging.debug('migrating %s to %s' % (sql_fields_old,sql_fields))
1014
1015 keys = sql_fields.keys()
1016 for key in sql_fields_old:
1017 if not key in keys:
1018 keys.append(key)
1019 new_add = self.concat_add(tablename)
1020
1021 metadata_change = False
1022 sql_fields_current = copy.copy(sql_fields_old)
1023 for key in keys:
1024 query = None
1025 if not key in sql_fields_old:
1026 sql_fields_current[key] = sql_fields[key]
1027 if self.dbengine in ('postgres',) and \
1028 sql_fields[key]['type'].startswith('geometry'):
1029
1030 query = [ sql_fields[key]['sql'] ]
1031 else:
1032 query = ['ALTER TABLE %s ADD %s %s;' % \
1033 (tablename, key,
1034 sql_fields_aux[key]['sql'].replace(', ', new_add))]
1035 metadata_change = True
1036 elif self.dbengine in ('sqlite', 'spatialite'):
1037 if key in sql_fields:
1038 sql_fields_current[key] = sql_fields[key]
1039 metadata_change = True
1040 elif not key in sql_fields:
1041 del sql_fields_current[key]
1042 ftype = sql_fields_old[key]['type']
1043 if self.dbengine in ('postgres',) \
1044 and ftype.startswith('geometry'):
1045 geotype, parms = ftype[:-1].split('(')
1046 schema = parms.split(',')[0]
1047 query = [ "SELECT DropGeometryColumn ('%(schema)s', '%(table)s', '%(field)s');" % dict(schema=schema, table=tablename, field=key,) ]
1048 elif not self.dbengine in ('firebird',):
1049 query = ['ALTER TABLE %s DROP COLUMN %s;'
1050 % (tablename, key)]
1051 else:
1052 query = ['ALTER TABLE %s DROP %s;' % (tablename, key)]
1053 metadata_change = True
1054 elif sql_fields[key]['sql'] != sql_fields_old[key]['sql'] \
1055 and not (key in table.fields and
1056 isinstance(table[key].type, SQLCustomType)) \
1057 and not sql_fields[key]['type'].startswith('reference')\
1058 and not sql_fields[key]['type'].startswith('double')\
1059 and not sql_fields[key]['type'].startswith('id'):
1060 sql_fields_current[key] = sql_fields[key]
1061 t = tablename
1062 tt = sql_fields_aux[key]['sql'].replace(', ', new_add)
1063 if not self.dbengine in ('firebird',):
1064 query = ['ALTER TABLE %s ADD %s__tmp %s;' % (t, key, tt),
1065 'UPDATE %s SET %s__tmp=%s;' % (t, key, key),
1066 'ALTER TABLE %s DROP COLUMN %s;' % (t, key),
1067 'ALTER TABLE %s ADD %s %s;' % (t, key, tt),
1068 'UPDATE %s SET %s=%s__tmp;' % (t, key, key),
1069 'ALTER TABLE %s DROP COLUMN %s__tmp;' % (t, key)]
1070 else:
1071 query = ['ALTER TABLE %s ADD %s__tmp %s;' % (t, key, tt),
1072 'UPDATE %s SET %s__tmp=%s;' % (t, key, key),
1073 'ALTER TABLE %s DROP %s;' % (t, key),
1074 'ALTER TABLE %s ADD %s %s;' % (t, key, tt),
1075 'UPDATE %s SET %s=%s__tmp;' % (t, key, key),
1076 'ALTER TABLE %s DROP %s__tmp;' % (t, key)]
1077 metadata_change = True
1078 elif sql_fields[key]['type'] != sql_fields_old[key]['type']:
1079 sql_fields_current[key] = sql_fields[key]
1080 metadata_change = True
1081
1082 if query:
1083 logfile.write('timestamp: %s\n'
1084 % datetime.datetime.today().isoformat())
1085 db['_lastsql'] = '\n'.join(query)
1086 for sub_query in query:
1087 logfile.write(sub_query + '\n')
1088 if not fake_migrate:
1089 self.execute(sub_query)
1090
1091
1092
1093 if db._adapter.commit_on_alter_table:
1094 db.commit()
1095 tfile = self.file_open(table._dbt, 'w')
1096 pickle.dump(sql_fields_current, tfile)
1097 self.file_close(tfile)
1098 logfile.write('success!\n')
1099 else:
1100 logfile.write('faked!\n')
1101 elif metadata_change:
1102 tfile = self.file_open(table._dbt, 'w')
1103 pickle.dump(sql_fields_current, tfile)
1104 self.file_close(tfile)
1105
1106 if metadata_change and \
1107 not (query and self.dbengine in ('mysql','oracle','firebird')):
1108 db.commit()
1109 tfile = self.file_open(table._dbt, 'w')
1110 pickle.dump(sql_fields_current, tfile)
1111 self.file_close(tfile)
1112
1113 - def LOWER(self, first):
1115
1116 - def UPPER(self, first):
1118
1119 - def COUNT(self, first, distinct=None):
1120 return ('COUNT(%s)' if not distinct else 'COUNT(DISTINCT %s)') \
1121 % self.expand(first)
1122
1124 return "EXTRACT(%s FROM %s)" % (what, self.expand(first))
1125
1126 - def EPOCH(self, first):
1128
1131
1134
1137
1140
1141 - def NOT_NULL(self, default, field_type):
1142 return 'NOT NULL DEFAULT %s' % self.represent(default,field_type)
1143
1145 expressions = [self.expand(first)]+[self.expand(e) for e in second]
1146 return 'COALESCE(%s)' % ','.join(expressions)
1147
1150
1151 - def RAW(self, first):
1153
1156
1158 return 'SUBSTR(%s,%s,%s)' % (self.expand(field), parameters[0], parameters[1])
1159
1161 return 'PRIMARY KEY(%s)' % key
1162
1163 - def _drop(self, table, mode):
1164 return ['DROP TABLE %s;' % table]
1165
1166 - def drop(self, table, mode=''):
1182
1183 - def _insert(self, table, fields):
1190
1192 return 'INSERT INTO %s DEFAULT VALUES;' % table
1193
1194 - def insert(self, table, fields):
1195 query = self._insert(table,fields)
1196 try:
1197 self.execute(query)
1198 except Exception:
1199 e = sys.exc_info()[1]
1200 if isinstance(e,self.integrity_error_class()):
1201 return None
1202 raise e
1203 if hasattr(table,'_primarykey'):
1204 return dict([(k[0].name, k[1]) for k in fields \
1205 if k[0].name in table._primarykey])
1206 id = self.lastrowid(table)
1207 if not isinstance(id,int):
1208 return id
1209 rid = Reference(id)
1210 (rid._table, rid._record) = (table, None)
1211 return rid
1212
1214 return [self.insert(table,item) for item in items]
1215
1216 - def NOT(self, first):
1218
1219 - def AND(self, first, second):
1221
1222 - def OR(self, first, second):
1224
1225 - def BELONGS(self, first, second):
1226 if isinstance(second, str):
1227 return '(%s IN (%s))' % (self.expand(first), second[:-1])
1228 elif not second:
1229 return '(1=0)'
1230 items = ','.join(self.expand(item, first.type) for item in second)
1231 return '(%s IN (%s))' % (self.expand(first), items)
1232
1233 - def REGEXP(self, first, second):
1234 "regular expression operator"
1235 raise NotImplementedError
1236
1237 - def LIKE(self, first, second):
1238 "case sensitive like operator"
1239 raise NotImplementedError
1240
1241 - def ILIKE(self, first, second):
1242 "case in-sensitive like operator"
1243 return '(%s LIKE %s)' % (self.expand(first),
1244 self.expand(second, 'string'))
1245
1247 return '(%s LIKE %s)' % (self.expand(first),
1248 self.expand(second+'%', 'string'))
1249
1251 return '(%s LIKE %s)' % (self.expand(first),
1252 self.expand('%'+second, 'string'))
1253
1254 - def CONTAINS(self, first, second, case_sensitive=False):
1255 if isinstance(second,Expression):
1256 field = self.expand(first)
1257 expr = self.expand(second,'string')
1258 if first.type.startswith('list:'):
1259 expr = 'CONCAT("|", %s, "|")' % expr
1260 elif not first.type in ('string', 'text', 'json'):
1261 raise RuntimeError("Expression Not Supported")
1262 return 'INSTR(%s,%s)' % (field, expr)
1263 else:
1264 if first.type in ('string', 'text', 'json'):
1265 key = '%'+str(second).replace('%','%%')+'%'
1266 elif first.type.startswith('list:'):
1267 key = '%|'+str(second).replace('|','||').replace('%','%%')+'|%'
1268 else:
1269 raise RuntimeError("Expression Not Supported")
1270 op = case_sensitive and self.LIKE or self.ILIKE
1271 return op(first,key)
1272
1273 - def EQ(self, first, second=None):
1278
1279 - def NE(self, first, second=None):
1284
1285 - def LT(self,first,second=None):
1286 if second is None:
1287 raise RuntimeError("Cannot compare %s < None" % first)
1288 return '(%s < %s)' % (self.expand(first),
1289 self.expand(second,first.type))
1290
1291 - def LE(self,first,second=None):
1292 if second is None:
1293 raise RuntimeError("Cannot compare %s <= None" % first)
1294 return '(%s <= %s)' % (self.expand(first),
1295 self.expand(second,first.type))
1296
1297 - def GT(self,first,second=None):
1298 if second is None:
1299 raise RuntimeError("Cannot compare %s > None" % first)
1300 return '(%s > %s)' % (self.expand(first),
1301 self.expand(second,first.type))
1302
1303 - def GE(self,first,second=None):
1304 if second is None:
1305 raise RuntimeError("Cannot compare %s >= None" % first)
1306 return '(%s >= %s)' % (self.expand(first),
1307 self.expand(second,first.type))
1308
1309 - def ADD(self, first, second):
1312
1313 - def SUB(self, first, second):
1316
1317 - def MUL(self, first, second):
1320
1321 - def DIV(self, first, second):
1324
1325 - def MOD(self, first, second):
1328
1329 - def AS(self, first, second):
1331
1332 - def ON(self, first, second):
1336
1339
1340 - def COMMA(self, first, second):
1342
1343 - def expand(self, expression, field_type=None):
1344 if isinstance(expression, Field):
1345 return '%s.%s' % (expression.tablename, expression.name)
1346 elif isinstance(expression, (Expression, Query)):
1347 first = expression.first
1348 second = expression.second
1349 op = expression.op
1350 optional_args = expression.optional_args or {}
1351 if not second is None:
1352 return op(first, second, **optional_args)
1353 elif not first is None:
1354 return op(first,**optional_args)
1355 elif isinstance(op, str):
1356 if op.endswith(';'):
1357 op=op[:-1]
1358 return '(%s)' % op
1359 else:
1360 return op()
1361 elif field_type:
1362 return str(self.represent(expression,field_type))
1363 elif isinstance(expression,(list,tuple)):
1364 return ','.join(self.represent(item,field_type) \
1365 for item in expression)
1366 elif isinstance(expression, bool):
1367 return '1' if expression else '0'
1368 else:
1369 return str(expression)
1370
1371 - def alias(self, table, alias):
1372 """
1373 Given a table object, makes a new table object
1374 with alias name.
1375 """
1376 other = copy.copy(table)
1377 other['_ot'] = other._tablename
1378 other['ALL'] = SQLALL(other)
1379 other['_tablename'] = alias
1380 for fieldname in other.fields:
1381 other[fieldname] = copy.copy(other[fieldname])
1382 other[fieldname]._tablename = alias
1383 other[fieldname].tablename = alias
1384 other[fieldname].table = other
1385 table._db[alias] = other
1386 return other
1387
1389 tablename = table._tablename
1390 return ['TRUNCATE TABLE %s %s;' % (tablename, mode or '')]
1391
1393
1394 if table._dbt:
1395 logfile = self.file_open(table._loggername, 'a')
1396 else:
1397 class Logfile(object):
1398 def write(self, value):
1399 pass
1400 def close(self):
1401 pass
1402 logfile = Logfile()
1403
1404 try:
1405 queries = table._db._adapter._truncate(table, mode)
1406 for query in queries:
1407 logfile.write(query + '\n')
1408 self.execute(query)
1409 table._db.commit()
1410 logfile.write('success!\n')
1411 finally:
1412 logfile.close()
1413
1414 - def _update(self, tablename, query, fields):
1415 if query:
1416 if use_common_filters(query):
1417 query = self.common_filter(query, [tablename])
1418 sql_w = ' WHERE ' + self.expand(query)
1419 else:
1420 sql_w = ''
1421 sql_v = ','.join(['%s=%s' % (field.name,
1422 self.expand(value, field.type)) \
1423 for (field, value) in fields])
1424 return 'UPDATE %s SET %s%s;' % (tablename, sql_v, sql_w)
1425
1426 - def update(self, tablename, query, fields):
1427 sql = self._update(tablename, query, fields)
1428 self.execute(sql)
1429 try:
1430 return self.cursor.rowcount
1431 except:
1432 return None
1433
1434 - def _delete(self, tablename, query):
1435 if query:
1436 if use_common_filters(query):
1437 query = self.common_filter(query, [tablename])
1438 sql_w = ' WHERE ' + self.expand(query)
1439 else:
1440 sql_w = ''
1441 return 'DELETE FROM %s%s;' % (tablename, sql_w)
1442
1443 - def delete(self, tablename, query):
1444 sql = self._delete(tablename, query)
1445
1446 db = self.db
1447 table = db[tablename]
1448 if self.dbengine in ('sqlite', 'spatialite') and table._referenced_by:
1449 deleted = [x[table._id.name] for x in db(query).select(table._id)]
1450
1451 self.execute(sql)
1452 try:
1453 counter = self.cursor.rowcount
1454 except:
1455 counter = None
1456
1457 if self.dbengine in ('sqlite', 'spatialite') and counter:
1458 for field in table._referenced_by:
1459 if field.type=='reference '+table._tablename \
1460 and field.ondelete=='CASCADE':
1461 db(field.belongs(deleted)).delete()
1462
1463 return counter
1464
1466 tablenames = self.tables(query)
1467 if len(tablenames)==1:
1468 return tablenames[0]
1469 elif len(tablenames)<1:
1470 raise RuntimeError("No table selected")
1471 else:
1472 raise RuntimeError("Too many tables selected")
1473
1475 db = self.db
1476 new_fields = []
1477 append = new_fields.append
1478 for item in fields:
1479 if isinstance(item,SQLALL):
1480 new_fields += item._table
1481 elif isinstance(item,str):
1482 if REGEX_TABLE_DOT_FIELD.match(item):
1483 tablename,fieldname = item.split('.')
1484 append(db[tablename][fieldname])
1485 else:
1486 append(Expression(db,lambda item=item:item))
1487 else:
1488 append(item)
1489
1490 if not new_fields:
1491 for table in tablenames:
1492 for field in db[table]:
1493 append(field)
1494 return new_fields
1495
1496 - def _select(self, query, fields, attributes):
1497 tables = self.tables
1498 for key in set(attributes.keys())-SELECT_ARGS:
1499 raise SyntaxError('invalid select attribute: %s' % key)
1500 args_get = attributes.get
1501 tablenames = tables(query)
1502 for field in fields:
1503 if isinstance(field, basestring) \
1504 and REGEX_TABLE_DOT_FIELD.match(field):
1505 tn,fn = field.split('.')
1506 field = self.db[tn][fn]
1507 for tablename in tables(field):
1508 if not tablename in tablenames:
1509 tablenames.append(tablename)
1510
1511 if len(tablenames) < 1:
1512 raise SyntaxError('Set: no tables selected')
1513 self._colnames = map(self.expand, fields)
1514 def geoexpand(field):
1515 if isinstance(field.type,str) and field.type.startswith('geometry'):
1516 field = field.st_astext()
1517 return self.expand(field)
1518 sql_f = ', '.join(map(geoexpand, fields))
1519 sql_o = ''
1520 sql_s = ''
1521 left = args_get('left', False)
1522 inner_join = args_get('join', False)
1523 distinct = args_get('distinct', False)
1524 groupby = args_get('groupby', False)
1525 orderby = args_get('orderby', False)
1526 having = args_get('having', False)
1527 limitby = args_get('limitby', False)
1528 for_update = args_get('for_update', False)
1529 if self.can_select_for_update is False and for_update is True:
1530 raise SyntaxError('invalid select attribute: for_update')
1531 if distinct is True:
1532 sql_s += 'DISTINCT'
1533 elif distinct:
1534 sql_s += 'DISTINCT ON (%s)' % distinct
1535 if inner_join:
1536 icommand = self.JOIN()
1537 if not isinstance(inner_join, (tuple, list)):
1538 inner_join = [inner_join]
1539 ijoint = [t._tablename for t in inner_join
1540 if not isinstance(t,Expression)]
1541 ijoinon = [t for t in inner_join if isinstance(t, Expression)]
1542 itables_to_merge={}
1543 [itables_to_merge.update(
1544 dict.fromkeys(tables(t))) for t in ijoinon]
1545 ijoinont = [t.first._tablename for t in ijoinon]
1546 [itables_to_merge.pop(t) for t in ijoinont
1547 if t in itables_to_merge]
1548 iimportant_tablenames = ijoint + ijoinont + itables_to_merge.keys()
1549 iexcluded = [t for t in tablenames
1550 if not t in iimportant_tablenames]
1551 if left:
1552 join = attributes['left']
1553 command = self.LEFT_JOIN()
1554 if not isinstance(join, (tuple, list)):
1555 join = [join]
1556 joint = [t._tablename for t in join
1557 if not isinstance(t, Expression)]
1558 joinon = [t for t in join if isinstance(t, Expression)]
1559
1560 tables_to_merge={}
1561 [tables_to_merge.update(
1562 dict.fromkeys(tables(t))) for t in joinon]
1563 joinont = [t.first._tablename for t in joinon]
1564 [tables_to_merge.pop(t) for t in joinont if t in tables_to_merge]
1565 important_tablenames = joint + joinont + tables_to_merge.keys()
1566 excluded = [t for t in tablenames
1567 if not t in important_tablenames ]
1568 else:
1569 excluded = tablenames
1570
1571 if use_common_filters(query):
1572 query = self.common_filter(query,excluded)
1573 sql_w = ' WHERE ' + self.expand(query) if query else ''
1574
1575 def alias(t):
1576 return str(self.db[t])
1577 if inner_join and not left:
1578 sql_t = ', '.join([alias(t) for t in iexcluded + \
1579 itables_to_merge.keys()])
1580 for t in ijoinon:
1581 sql_t += ' %s %s' % (icommand, str(t))
1582 elif not inner_join and left:
1583 sql_t = ', '.join([alias(t) for t in excluded + \
1584 tables_to_merge.keys()])
1585 if joint:
1586 sql_t += ' %s %s' % (command, ','.join([t for t in joint]))
1587 for t in joinon:
1588 sql_t += ' %s %s' % (command, str(t))
1589 elif inner_join and left:
1590 all_tables_in_query = set(important_tablenames + \
1591 iimportant_tablenames + \
1592 tablenames)
1593 tables_in_joinon = set(joinont + ijoinont)
1594 tables_not_in_joinon = \
1595 all_tables_in_query.difference(tables_in_joinon)
1596 sql_t = ','.join([alias(t) for t in tables_not_in_joinon])
1597 for t in ijoinon:
1598 sql_t += ' %s %s' % (icommand, str(t))
1599 if joint:
1600 sql_t += ' %s %s' % (command, ','.join([t for t in joint]))
1601 for t in joinon:
1602 sql_t += ' %s %s' % (command, str(t))
1603 else:
1604 sql_t = ', '.join(alias(t) for t in tablenames)
1605 if groupby:
1606 if isinstance(groupby, (list, tuple)):
1607 groupby = xorify(groupby)
1608 sql_o += ' GROUP BY %s' % self.expand(groupby)
1609 if having:
1610 sql_o += ' HAVING %s' % attributes['having']
1611 if orderby:
1612 if isinstance(orderby, (list, tuple)):
1613 orderby = xorify(orderby)
1614 if str(orderby) == '<random>':
1615 sql_o += ' ORDER BY %s' % self.RANDOM()
1616 else:
1617 sql_o += ' ORDER BY %s' % self.expand(orderby)
1618 if limitby:
1619 if not orderby and tablenames:
1620 sql_o += ' ORDER BY %s' % ', '.join(['%s.%s'%(t,x) for t in tablenames for x in (hasattr(self.db[t],'_primarykey') and self.db[t]._primarykey or [self.db[t]._id.name])])
1621
1622 sql = self.select_limitby(sql_s, sql_f, sql_t, sql_w, sql_o, limitby)
1623 if for_update and self.can_select_for_update is True:
1624 sql = sql.rstrip(';') + ' FOR UPDATE;'
1625 return sql
1626
1627 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
1628 if limitby:
1629 (lmin, lmax) = limitby
1630 sql_o += ' LIMIT %i OFFSET %i' % (lmax - lmin, lmin)
1631 return 'SELECT %s %s FROM %s%s%s;' % \
1632 (sql_s, sql_f, sql_t, sql_w, sql_o)
1633
1635 return self.cursor.fetchall()
1636
1638 args_get = attributes.get
1639 cache = args_get('cache',None)
1640 if not cache:
1641 self.execute(sql)
1642 rows = self._fetchall()
1643 else:
1644 (cache_model, time_expire) = cache
1645 key = self.uri + '/' + sql + '/rows'
1646 if len(key)>200: key = hashlib_md5(key).hexdigest()
1647 def _select_aux2():
1648 self.execute(sql)
1649 return self._fetchall()
1650 rows = cache_model(key,_select_aux2,time_expire)
1651 if isinstance(rows,tuple):
1652 rows = list(rows)
1653 limitby = args_get('limitby', None) or (0,)
1654 rows = self.rowslice(rows,limitby[0],None)
1655 processor = args_get('processor',self.parse)
1656 cacheable = args_get('cacheable',False)
1657 return processor(rows,fields,self._colnames,cacheable=cacheable)
1658
1659 - def select(self, query, fields, attributes):
1660 """
1661 Always returns a Rows object, possibly empty.
1662 """
1663 sql = self._select(query, fields, attributes)
1664 cache = attributes.get('cache', None)
1665 if cache and attributes.get('cacheable',False):
1666 del attributes['cache']
1667 (cache_model, time_expire) = cache
1668 key = self.uri + '/' + sql
1669 if len(key)>200: key = hashlib_md5(key).hexdigest()
1670 args = (sql,fields,attributes)
1671 return cache_model(
1672 key,
1673 lambda self=self,args=args:self._select_aux(*args),
1674 time_expire)
1675 else:
1676 return self._select_aux(sql,fields,attributes)
1677
1678 - def _count(self, query, distinct=None):
1679 tablenames = self.tables(query)
1680 if query:
1681 if use_common_filters(query):
1682 query = self.common_filter(query, tablenames)
1683 sql_w = ' WHERE ' + self.expand(query)
1684 else:
1685 sql_w = ''
1686 sql_t = ','.join(tablenames)
1687 if distinct:
1688 if isinstance(distinct,(list, tuple)):
1689 distinct = xorify(distinct)
1690 sql_d = self.expand(distinct)
1691 return 'SELECT count(DISTINCT %s) FROM %s%s;' % \
1692 (sql_d, sql_t, sql_w)
1693 return 'SELECT count(*) FROM %s%s;' % (sql_t, sql_w)
1694
1695 - def count(self, query, distinct=None):
1696 self.execute(self._count(query, distinct))
1697 return self.cursor.fetchone()[0]
1698
1710
1713
1716
1719
1722
1725
1728
1731
1734
1736 return '%s_%s__constraint' % (table,fieldname)
1737
1740
1742 if not self.connection: return None
1743 command = a[0]
1744 if self.db._debug:
1745 LOGGER.debug('SQL: %s' % command)
1746 self.db._lastsql = command
1747 t0 = time.time()
1748 ret = self.cursor.execute(*a, **b)
1749 self.db._timings.append((command,time.time()-t0))
1750 del self.db._timings[:-TIMINGSSIZE]
1751 return ret
1752
1755
1757 field_is_type = fieldtype.startswith
1758 if isinstance(obj, CALLABLETYPES):
1759 obj = obj()
1760 if isinstance(fieldtype, SQLCustomType):
1761 value = fieldtype.encoder(obj)
1762 if fieldtype.type in ('string','text', 'json'):
1763 return self.adapt(value)
1764 return value
1765 if isinstance(obj, (Expression, Field)):
1766 return str(obj)
1767 if field_is_type('list:'):
1768 if not obj:
1769 obj = []
1770 elif not isinstance(obj, (list, tuple)):
1771 obj = [obj]
1772 if field_is_type('list:string'):
1773 obj = map(str,obj)
1774 else:
1775 obj = map(int,obj)
1776
1777 if isinstance(obj, (list, tuple)) and (not fieldtype == "json"):
1778 obj = bar_encode(obj)
1779 if obj is None:
1780 return 'NULL'
1781 if obj == '' and not fieldtype[:2] in ['st', 'te', 'js', 'pa', 'up']:
1782 return 'NULL'
1783 r = self.represent_exceptions(obj, fieldtype)
1784 if not r is None:
1785 return r
1786 if fieldtype == 'boolean':
1787 if obj and not str(obj)[:1].upper() in '0F':
1788 return self.smart_adapt(self.TRUE)
1789 else:
1790 return self.smart_adapt(self.FALSE)
1791 if fieldtype == 'id' or fieldtype == 'integer':
1792 return str(int(obj))
1793 if field_is_type('decimal'):
1794 return str(obj)
1795 elif field_is_type('reference'):
1796 if fieldtype.find('.')>0:
1797 return repr(obj)
1798 elif isinstance(obj, (Row, Reference)):
1799 return str(obj['id'])
1800 return str(int(obj))
1801 elif fieldtype == 'double':
1802 return repr(float(obj))
1803 if isinstance(obj, unicode):
1804 obj = obj.encode(self.db_codec)
1805 if fieldtype == 'blob':
1806 obj = base64.b64encode(str(obj))
1807 elif fieldtype == 'date':
1808 if isinstance(obj, (datetime.date, datetime.datetime)):
1809 obj = obj.isoformat()[:10]
1810 else:
1811 obj = str(obj)
1812 elif fieldtype == 'datetime':
1813 if isinstance(obj, datetime.datetime):
1814 obj = obj.isoformat(self.T_SEP)[:19]
1815 elif isinstance(obj, datetime.date):
1816 obj = obj.isoformat()[:10]+' 00:00:00'
1817 else:
1818 obj = str(obj)
1819 elif fieldtype == 'time':
1820 if isinstance(obj, datetime.time):
1821 obj = obj.isoformat()[:10]
1822 else:
1823 obj = str(obj)
1824 elif fieldtype == 'json':
1825 if not self.native_json:
1826 if have_serializers:
1827 obj = serializers.json(obj)
1828 elif simplejson:
1829 obj = simplejson.dumps(items)
1830 else:
1831 raise RuntimeError("missing simplejson")
1832 if not isinstance(obj,bytes):
1833 obj = bytes(obj)
1834 try:
1835 obj.decode(self.db_codec)
1836 except:
1837 obj = obj.decode('latin1').encode(self.db_codec)
1838 return self.adapt(obj)
1839
1842
1845
1848
1849 - def rowslice(self, rows, minimum=0, maximum=None):
1850 """
1851 By default this function does nothing;
1852 overload when db does not do slicing.
1853 """
1854 return rows
1855
1856 - def parse_value(self, value, field_type, blob_decode=True):
1857 if field_type != 'blob' and isinstance(value, str):
1858 try:
1859 value = value.decode(self.db._db_codec)
1860 except Exception:
1861 pass
1862 if isinstance(value, unicode):
1863 value = value.encode('utf-8')
1864 if isinstance(field_type, SQLCustomType):
1865 value = field_type.decoder(value)
1866 if not isinstance(field_type, str) or value is None:
1867 return value
1868 elif field_type in ('string', 'text', 'password', 'upload', 'dict'):
1869 return value
1870 elif field_type.startswith('geo'):
1871 return value
1872 elif field_type == 'blob' and not blob_decode:
1873 return value
1874 else:
1875 key = REGEX_TYPE.match(field_type).group(0)
1876 return self.parsemap[key](value,field_type)
1877
1879 referee = field_type[10:].strip()
1880 if not '.' in referee:
1881 value = Reference(value)
1882 value._table, value._record = self.db[referee], None
1883 return value
1884
1886 return value == True or str(value)[:1].lower() == 't'
1887
1889 if isinstance(value, datetime.datetime):
1890 return value.date()
1891 if not isinstance(value, (datetime.date,datetime.datetime)):
1892 (y, m, d) = map(int, str(value)[:10].strip().split('-'))
1893 value = datetime.date(y, m, d)
1894 return value
1895
1897 if not isinstance(value, datetime.time):
1898 time_items = map(int,str(value)[:8].strip().split(':')[:3])
1899 if len(time_items) == 3:
1900 (h, mi, s) = time_items
1901 else:
1902 (h, mi, s) = time_items + [0]
1903 value = datetime.time(h, mi, s)
1904 return value
1905
1907 if not isinstance(value, datetime.datetime):
1908 value = str(value)
1909 date_part,time_part,timezone = value[:10],value[11:19],value[19:]
1910 if '+' in timezone:
1911 ms,tz = timezone.split('+')
1912 h,m = tz.split(':')
1913 dt = datetime.timedelta(seconds=3600*int(h)+60*int(m))
1914 elif '-' in timezone:
1915 ms,tz = timezone.split('-')
1916 h,m = tz.split(':')
1917 dt = -datetime.timedelta(seconds=3600*int(h)+60*int(m))
1918 else:
1919 dt = None
1920 (y, m, d) = map(int,date_part.split('-'))
1921 time_parts = time_part and time_part.split(':')[:3] or (0,0,0)
1922 while len(time_parts)<3: time_parts.append(0)
1923 time_items = map(int,time_parts)
1924 (h, mi, s) = time_items
1925 value = datetime.datetime(y, m, d, h, mi, s)
1926 if dt:
1927 value = value + dt
1928 return value
1929
1931 return base64.b64decode(str(value))
1932
1934 decimals = int(field_type[8:-1].split(',')[-1])
1935 if self.dbengine in ('sqlite', 'spatialite'):
1936 value = ('%.' + str(decimals) + 'f') % value
1937 if not isinstance(value, decimal.Decimal):
1938 value = decimal.Decimal(str(value))
1939 return value
1940
1945
1950
1955
1956 - def parse_id(self, value, field_type):
1958
1961
1964
1966 if not self.native_json:
1967 if not isinstance(value, basestring):
1968 raise RuntimeError('json data not a string')
1969 if isinstance(value, unicode):
1970 value = value.encode('utf-8')
1971 if have_serializers:
1972 value = serializers.loads_json(value)
1973 elif simplejson:
1974 value = simplejson.loads(value)
1975 else:
1976 raise RuntimeError("missing simplejson")
1977 return value
1978
1980 self.parsemap = {
1981 'id':self.parse_id,
1982 'integer':self.parse_integer,
1983 'bigint':self.parse_integer,
1984 'float':self.parse_double,
1985 'double':self.parse_double,
1986 'reference':self.parse_reference,
1987 'boolean':self.parse_boolean,
1988 'date':self.parse_date,
1989 'time':self.parse_time,
1990 'datetime':self.parse_datetime,
1991 'blob':self.parse_blob,
1992 'decimal':self.parse_decimal,
1993 'json':self.parse_json,
1994 'list:integer':self.parse_list_integers,
1995 'list:reference':self.parse_list_references,
1996 'list:string':self.parse_list_strings,
1997 }
1998
1999 - def parse(self, rows, fields, colnames, blob_decode=True,
2000 cacheable = False):
2001 self.build_parsemap()
2002 db = self.db
2003 virtualtables = []
2004 new_rows = []
2005 tmps = []
2006 for colname in colnames:
2007 if not REGEX_TABLE_DOT_FIELD.match(colname):
2008 tmps.append(None)
2009 else:
2010 (tablename, fieldname) = colname.split('.')
2011 table = db[tablename]
2012 field = table[fieldname]
2013 ft = field.type
2014 tmps.append((tablename,fieldname,table,field,ft))
2015 for (i,row) in enumerate(rows):
2016 new_row = Row()
2017 for (j,colname) in enumerate(colnames):
2018 value = row[j]
2019 tmp = tmps[j]
2020 if tmp:
2021 (tablename,fieldname,table,field,ft) = tmp
2022 if tablename in new_row:
2023 colset = new_row[tablename]
2024 else:
2025 colset = new_row[tablename] = Row()
2026 if tablename not in virtualtables:
2027 virtualtables.append(tablename)
2028 value = self.parse_value(value,ft,blob_decode)
2029 if field.filter_out:
2030 value = field.filter_out(value)
2031 colset[fieldname] = value
2032
2033
2034 if ft=='id' and fieldname!='id' and \
2035 not 'id' in table.fields:
2036 colset['id'] = value
2037
2038 if ft == 'id' and not cacheable:
2039
2040
2041
2042 if isinstance(self, GoogleDatastoreAdapter):
2043 id = value.key().id_or_name()
2044 colset[fieldname] = id
2045 colset.gae_item = value
2046 else:
2047 id = value
2048 colset.update_record = RecordUpdater(colset,table,id)
2049 colset.delete_record = RecordDeleter(table,id)
2050 for rfield in table._referenced_by:
2051 referee_link = db._referee_name and \
2052 db._referee_name % dict(
2053 table=rfield.tablename,field=rfield.name)
2054 if referee_link and not referee_link in colset:
2055 colset[referee_link] = LazySet(rfield,id)
2056 else:
2057 if not '_extra' in new_row:
2058 new_row['_extra'] = Row()
2059 new_row['_extra'][colname] = \
2060 self.parse_value(value,
2061 fields[j].type,blob_decode)
2062 new_column_name = \
2063 REGEX_SELECT_AS_PARSER.search(colname)
2064 if not new_column_name is None:
2065 column_name = new_column_name.groups(0)
2066 setattr(new_row,column_name[0],value)
2067 new_rows.append(new_row)
2068 rowsobj = Rows(db, new_rows, colnames, rawrows=rows)
2069
2070 for tablename in virtualtables:
2071
2072 table = db[tablename]
2073 fields_virtual = [(f,v) for (f,v) in table.iteritems()
2074 if isinstance(v,FieldVirtual)]
2075 fields_lazy = [(f,v) for (f,v) in table.iteritems()
2076 if isinstance(v,FieldMethod)]
2077 if fields_virtual or fields_lazy:
2078 for row in rowsobj.records:
2079 box = row[tablename]
2080 for f,v in fields_virtual:
2081 box[f] = v.f(row)
2082 for f,v in fields_lazy:
2083 box[f] = (v.handler or VirtualCommand)(v.f,row)
2084
2085
2086 for item in table.virtualfields:
2087 try:
2088 rowsobj = rowsobj.setvirtualfields(**{tablename:item})
2089 except (KeyError, AttributeError):
2090
2091 pass
2092 return rowsobj
2093
2095 tenant_fieldname = self.db._request_tenant
2096
2097 for tablename in tablenames:
2098 table = self.db[tablename]
2099
2100
2101 if table._common_filter != None:
2102 query = query & table._common_filter(query)
2103
2104
2105 if tenant_fieldname in table:
2106 default = table[tenant_fieldname].default
2107 if not default is None:
2108 newquery = table[tenant_fieldname] == default
2109 if query is None:
2110 query = newquery
2111 else:
2112 query = query & newquery
2113 return query
2114
2115 - def CASE(self,query,t,f):
2116 def represent(x):
2117 types = {type(True):'boolean',type(0):'integer',type(1.0):'double'}
2118 if x is None: return 'NULL'
2119 elif isinstance(x,Expression): return str(x)
2120 else: return self.represent(x,types.get(type(x),'string'))
2121 return Expression(self.db,'CASE WHEN %s THEN %s ELSE %s END' % \
2122 (self.expand(query),represent(t),represent(f)))
2123
2129 drivers = ('sqlite2','sqlite3')
2130
2131 can_select_for_update = None
2132
2134 return "web2py_extract('%s',%s)" % (what, self.expand(field))
2135
2136 @staticmethod
2138 table = {
2139 'year': (0, 4),
2140 'month': (5, 7),
2141 'day': (8, 10),
2142 'hour': (11, 13),
2143 'minute': (14, 16),
2144 'second': (17, 19),
2145 }
2146 try:
2147 if lookup != 'epoch':
2148 (i, j) = table[lookup]
2149 return int(s[i:j])
2150 else:
2151 return time.mktime(datetime.datetime.strptime(s, '%Y-%m-%d %H:%M:%S').timetuple())
2152 except:
2153 return None
2154
2155 @staticmethod
2157 return re.compile(expression).search(item) is not None
2158
2159 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec ='UTF-8',
2160 credential_decoder=IDENTITY, driver_args={},
2161 adapter_args={}, do_connect=True, after_connection=None):
2162 self.db = db
2163 self.dbengine = "sqlite"
2164 self.uri = uri
2165 if do_connect: self.find_driver(adapter_args)
2166 self.pool_size = 0
2167 self.folder = folder
2168 self.db_codec = db_codec
2169 self._after_connection = after_connection
2170 self.find_or_make_work_folder()
2171 path_encoding = sys.getfilesystemencoding() \
2172 or locale.getdefaultlocale()[1] or 'utf8'
2173 if uri.startswith('sqlite:memory'):
2174 dbpath = ':memory:'
2175 else:
2176 dbpath = uri.split('://',1)[1]
2177 if dbpath[0] != '/':
2178 if PYTHON_VERSION == 2:
2179 dbpath = pjoin(
2180 self.folder.decode(path_encoding).encode('utf8'), dbpath)
2181 else:
2182 dbpath = pjoin(self.folder, dbpath)
2183 if not 'check_same_thread' in driver_args:
2184 driver_args['check_same_thread'] = False
2185 if not 'detect_types' in driver_args and do_connect:
2186 driver_args['detect_types'] = self.driver.PARSE_DECLTYPES
2187 def connector(dbpath=dbpath, driver_args=driver_args):
2188 return self.driver.Connection(dbpath, **driver_args)
2189 self.connector = connector
2190 if do_connect: self.reconnect()
2191
2197
2199 tablename = table._tablename
2200 return ['DELETE FROM %s;' % tablename,
2201 "DELETE FROM sqlite_sequence WHERE name='%s';" % tablename]
2202
2205
2206 - def REGEXP(self,first,second):
2207 return '(%s REGEXP %s)' % (self.expand(first),
2208 self.expand(second,'string'))
2209
2210 - def select(self, query, fields, attributes):
2211 """
2212 Simulate SELECT ... FOR UPDATE with BEGIN IMMEDIATE TRANSACTION.
2213 Note that the entire database, rather than one record, is locked
2214 (it will be locked eventually anyway by the following UPDATE).
2215 """
2216 if attributes.get('for_update', False) and not 'cache' in attributes:
2217 self.execute('BEGIN IMMEDIATE TRANSACTION;')
2218 return super(SQLiteAdapter, self).select(query, fields, attributes)
2219
2221 drivers = ('sqlite3','sqlite2')
2222
2223 types = copy.copy(BaseAdapter.types)
2224 types.update(geometry='GEOMETRY')
2225
2226 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec ='UTF-8',
2227 credential_decoder=IDENTITY, driver_args={},
2228 adapter_args={}, do_connect=True, srid=4326, after_connection=None):
2229 self.db = db
2230 self.dbengine = "spatialite"
2231 self.uri = uri
2232 if do_connect: self.find_driver(adapter_args)
2233 self.pool_size = 0
2234 self.folder = folder
2235 self.db_codec = db_codec
2236 self._after_connection = after_connection
2237 self.find_or_make_work_folder()
2238 self.srid = srid
2239 path_encoding = sys.getfilesystemencoding() \
2240 or locale.getdefaultlocale()[1] or 'utf8'
2241 if uri.startswith('spatialite:memory'):
2242 dbpath = ':memory:'
2243 else:
2244 dbpath = uri.split('://',1)[1]
2245 if dbpath[0] != '/':
2246 dbpath = pjoin(
2247 self.folder.decode(path_encoding).encode('utf8'), dbpath)
2248 if not 'check_same_thread' in driver_args:
2249 driver_args['check_same_thread'] = False
2250 if not 'detect_types' in driver_args and do_connect:
2251 driver_args['detect_types'] = self.driver.PARSE_DECLTYPES
2252 def connector(dbpath=dbpath, driver_args=driver_args):
2253 return self.driver.Connection(dbpath, **driver_args)
2254 self.connector = connector
2255 if do_connect: self.reconnect()
2256
2269
2270
2271
2273 return 'AsGeoJSON(%s,%s,%s)' %(self.expand(first),
2274 second['precision'], second['options'])
2275
2276 - def ST_ASTEXT(self, first):
2277 return 'AsText(%s)' %(self.expand(first))
2278
2282
2286
2290
2294
2298
2300 return 'Simplify(%s,%s)' %(self.expand(first),
2301 self.expand(second, 'double'))
2302
2306
2310
2312 field_is_type = fieldtype.startswith
2313 if field_is_type('geo'):
2314 srid = 4326
2315 geotype, parms = fieldtype[:-1].split('(')
2316 parms = parms.split(',')
2317 if len(parms) >= 2:
2318 schema, srid = parms[:2]
2319
2320 value = "ST_GeomFromText('%s',%s)" %(obj, srid)
2321
2322
2323
2324
2325 return value
2326 return BaseAdapter.represent(self, obj, fieldtype)
2327
2330 drivers = ('zxJDBC_sqlite',)
2331
2332 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec='UTF-8',
2333 credential_decoder=IDENTITY, driver_args={},
2334 adapter_args={}, do_connect=True, after_connection=None):
2335 self.db = db
2336 self.dbengine = "sqlite"
2337 self.uri = uri
2338 if do_connect: self.find_driver(adapter_args)
2339 self.pool_size = pool_size
2340 self.folder = folder
2341 self.db_codec = db_codec
2342 self._after_connection = after_connection
2343 self.find_or_make_work_folder()
2344 path_encoding = sys.getfilesystemencoding() \
2345 or locale.getdefaultlocale()[1] or 'utf8'
2346 if uri.startswith('sqlite:memory'):
2347 dbpath = ':memory:'
2348 else:
2349 dbpath = uri.split('://',1)[1]
2350 if dbpath[0] != '/':
2351 dbpath = pjoin(
2352 self.folder.decode(path_encoding).encode('utf8'), dbpath)
2353 def connector(dbpath=dbpath,driver_args=driver_args):
2354 return self.driver.connect(
2355 self.driver.getConnection('jdbc:sqlite:'+dbpath),
2356 **driver_args)
2357 self.connector = connector
2358 if do_connect: self.reconnect()
2359
2364
2367
2370 drivers = ('MySQLdb','pymysql')
2371
2372 maxcharlength = 255
2373 commit_on_alter_table = True
2374 support_distributed_transaction = True
2375 types = {
2376 'boolean': 'CHAR(1)',
2377 'string': 'VARCHAR(%(length)s)',
2378 'text': 'LONGTEXT',
2379 'json': 'LONGTEXT',
2380 'password': 'VARCHAR(%(length)s)',
2381 'blob': 'LONGBLOB',
2382 'upload': 'VARCHAR(%(length)s)',
2383 'integer': 'INT',
2384 'bigint': 'BIGINT',
2385 'float': 'FLOAT',
2386 'double': 'DOUBLE',
2387 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2388 'date': 'DATE',
2389 'time': 'TIME',
2390 'datetime': 'DATETIME',
2391 'id': 'INT AUTO_INCREMENT NOT NULL',
2392 'reference': 'INT, INDEX %(index_name)s (%(field_name)s), FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2393 'list:integer': 'LONGTEXT',
2394 'list:string': 'LONGTEXT',
2395 'list:reference': 'LONGTEXT',
2396 'big-id': 'BIGINT AUTO_INCREMENT NOT NULL',
2397 'big-reference': 'BIGINT, INDEX %(index_name)s (%(field_name)s), FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2398 }
2399
2402
2405
2407 return 'SUBSTRING(%s,%s,%s)' % (self.expand(field),
2408 parameters[0], parameters[1])
2409
2410 - def EPOCH(self, first):
2412
2413 - def REGEXP(self,first,second):
2414 return '(%s REGEXP %s)' % (self.expand(first),
2415 self.expand(second,'string'))
2416
2417 - def _drop(self,table,mode):
2418
2419 return ['SET FOREIGN_KEY_CHECKS=0;','DROP TABLE %s;' % table,
2420 'SET FOREIGN_KEY_CHECKS=1;']
2421
2423 return 'INSERT INTO %s VALUES (DEFAULT);' % table
2424
2427
2431
2434
2437
2438 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^?]+)(\?set_encoding=(?P<charset>\w+))?$')
2439
2440 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
2441 credential_decoder=IDENTITY, driver_args={},
2442 adapter_args={}, do_connect=True, after_connection=None):
2443 self.db = db
2444 self.dbengine = "mysql"
2445 self.uri = uri
2446 if do_connect: self.find_driver(adapter_args,uri)
2447 self.pool_size = pool_size
2448 self.folder = folder
2449 self.db_codec = db_codec
2450 self._after_connection = after_connection
2451 self.find_or_make_work_folder()
2452 ruri = uri.split('://',1)[1]
2453 m = self.REGEX_URI.match(ruri)
2454 if not m:
2455 raise SyntaxError(
2456 "Invalid URI string in DAL: %s" % self.uri)
2457 user = credential_decoder(m.group('user'))
2458 if not user:
2459 raise SyntaxError('User required')
2460 password = credential_decoder(m.group('password'))
2461 if not password:
2462 password = ''
2463 host = m.group('host')
2464 if not host:
2465 raise SyntaxError('Host name required')
2466 db = m.group('db')
2467 if not db:
2468 raise SyntaxError('Database name required')
2469 port = int(m.group('port') or '3306')
2470 charset = m.group('charset') or 'utf8'
2471 driver_args.update(db=db,
2472 user=credential_decoder(user),
2473 passwd=credential_decoder(password),
2474 host=host,
2475 port=port,
2476 charset=charset)
2477
2478
2479 def connector(driver_args=driver_args):
2480 return self.driver.connect(**driver_args)
2481 self.connector = connector
2482 if do_connect: self.reconnect()
2483
2485 self.execute('SET FOREIGN_KEY_CHECKS=1;')
2486 self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
2487
2489 self.execute('select last_insert_id();')
2490 return int(self.cursor.fetchone()[0])
2491
2492 -class PostgreSQLAdapter(BaseAdapter):
2493 drivers = ('psycopg2','pg8000')
2494
2495 support_distributed_transaction = True
2496 types = {
2497 'boolean': 'CHAR(1)',
2498 'string': 'VARCHAR(%(length)s)',
2499 'text': 'TEXT',
2500 'json': 'TEXT',
2501 'password': 'VARCHAR(%(length)s)',
2502 'blob': 'BYTEA',
2503 'upload': 'VARCHAR(%(length)s)',
2504 'integer': 'INTEGER',
2505 'bigint': 'BIGINT',
2506 'float': 'FLOAT',
2507 'double': 'FLOAT8',
2508 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2509 'date': 'DATE',
2510 'time': 'TIME',
2511 'datetime': 'TIMESTAMP',
2512 'id': 'SERIAL PRIMARY KEY',
2513 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2514 'list:integer': 'TEXT',
2515 'list:string': 'TEXT',
2516 'list:reference': 'TEXT',
2517 'geometry': 'GEOMETRY',
2518 'geography': 'GEOGRAPHY',
2519 'big-id': 'BIGSERIAL PRIMARY KEY',
2520 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2521 }
2522
2523 - def varquote(self,name):
2524 return varquote_aux(name,'"%s"')
2525
2526 - def adapt(self,obj):
2527 if self.driver_name == 'psycopg2':
2528 return psycopg2_adapt(obj).getquoted()
2529 elif self.driver_name == 'pg8000':
2530 return "'%s'" % str(obj).replace("%","%%").replace("'","''")
2531 else:
2532 return "'%s'" % str(obj).replace("'","''")
2533
2534 - def sequence_name(self,table):
2535 return '%s_id_Seq' % table
2536
2539
2540 - def ADD(self, first, second):
2541 t = first.type
2542 if t in ('text','string','password', 'json', 'upload','blob'):
2543 return '(%s || %s)' % (self.expand(first), self.expand(second, t))
2544 else:
2545 return '(%s + %s)' % (self.expand(first), self.expand(second, t))
2546
2549
2550 - def prepare(self,key):
2551 self.execute("PREPARE TRANSACTION '%s';" % key)
2552
2553 - def commit_prepared(self,key):
2554 self.execute("COMMIT PREPARED '%s';" % key)
2555
2556 - def rollback_prepared(self,key):
2557 self.execute("ROLLBACK PREPARED '%s';" % key)
2558
2559 - def create_sequence_and_triggers(self, query, table, **args):
2560
2561
2562
2563
2564 self.execute(query)
2565
2566 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?sslmode=(?P<sslmode>.+))?$')
2567
2568 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
2569 credential_decoder=IDENTITY, driver_args={},
2570 adapter_args={}, do_connect=True, srid=4326,
2571 after_connection=None):
2572 self.db = db
2573 self.dbengine = "postgres"
2574 self.uri = uri
2575 if do_connect: self.find_driver(adapter_args,uri)
2576 self.pool_size = pool_size
2577 self.folder = folder
2578 self.db_codec = db_codec
2579 self._after_connection = after_connection
2580 self.srid = srid
2581 self.find_or_make_work_folder()
2582 ruri = uri.split('://',1)[1]
2583 m = self.REGEX_URI.match(ruri)
2584 if not m:
2585 raise SyntaxError("Invalid URI string in DAL")
2586 user = credential_decoder(m.group('user'))
2587 if not user:
2588 raise SyntaxError('User required')
2589 password = credential_decoder(m.group('password'))
2590 if not password:
2591 password = ''
2592 host = m.group('host')
2593 if not host:
2594 raise SyntaxError('Host name required')
2595 db = m.group('db')
2596 if not db:
2597 raise SyntaxError('Database name required')
2598 port = m.group('port') or '5432'
2599 sslmode = m.group('sslmode')
2600 if sslmode:
2601 msg = ("dbname='%s' user='%s' host='%s' "
2602 "port=%s password='%s' sslmode='%s'") \
2603 % (db, user, host, port, password, sslmode)
2604 else:
2605 msg = ("dbname='%s' user='%s' host='%s' "
2606 "port=%s password='%s'") \
2607 % (db, user, host, port, password)
2608
2609 self.__version__ = "%s %s" % (self.driver.__name__, self.driver.__version__)
2610 def connector(msg=msg,driver_args=driver_args):
2611 return self.driver.connect(msg,**driver_args)
2612 self.connector = connector
2613 if do_connect: self.reconnect()
2614
2615 - def after_connection(self):
2616 self.connection.set_client_encoding('UTF8')
2617 self.execute("SET standard_conforming_strings=on;")
2618 self.try_json()
2619
2620 - def lastrowid(self,table):
2621 self.execute("select currval('%s')" % table._sequence_name)
2622 return int(self.cursor.fetchone()[0])
2623
2624 - def try_json(self):
2625
2626
2627 if self.driver_name == "pg8000":
2628 supports_json = self.connection.server_version >= "9.2.0"
2629 elif (self.driver_name == "psycopg2") and \
2630 (self.driver.__version__ >= "2.0.12"):
2631 supports_json = self.connection.server_version >= 90200
2632 elif self.driver_name == "zxJDBC":
2633 supports_json = self.connection.dbversion >= "9.2.0"
2634 else: supports_json = None
2635 if supports_json: self.types["json"] = "JSON"
2636 else: LOGGER.debug("Your database version does not support the JSON data type (using TEXT instead)")
2637
2638 - def LIKE(self,first,second):
2639 args = (self.expand(first), self.expand(second,'string'))
2640 if not first.type in ('string', 'text', 'json'):
2641 return '(CAST(%s AS CHAR(%s)) LIKE %s)' % (args[0], first.length, args[1])
2642 else:
2643 return '(%s LIKE %s)' % args
2644
2645 - def ILIKE(self,first,second):
2646 args = (self.expand(first), self.expand(second,'string'))
2647 if not first.type in ('string', 'text', 'json'):
2648 return '(CAST(%s AS CHAR(%s)) LIKE %s)' % (args[0], first.length, args[1])
2649 else:
2650 return '(%s ILIKE %s)' % args
2651
2652 - def REGEXP(self,first,second):
2653 return '(%s ~ %s)' % (self.expand(first),
2654 self.expand(second,'string'))
2655
2656 - def STARTSWITH(self,first,second):
2657 return '(%s ILIKE %s)' % (self.expand(first),
2658 self.expand(second+'%','string'))
2659
2660 - def ENDSWITH(self,first,second):
2661 return '(%s ILIKE %s)' % (self.expand(first),
2662 self.expand('%'+second,'string'))
2663
2664 - def CONTAINS(self,first,second,case_sensitive=False):
2665 if first.type in ('string','text', 'json'):
2666 second = '%'+str(second).replace('%','%%')+'%'
2667 elif first.type.startswith('list:'):
2668 second = '%|'+str(second).replace('|','||').replace('%','%%')+'|%'
2669 op = case_sensitive and self.LIKE or self.ILIKE
2670 return op(first,second)
2671
2672
2673
2674 - def ST_ASGEOJSON(self, first, second):
2675 """
2676 http://postgis.org/docs/ST_AsGeoJSON.html
2677 """
2678 return 'ST_AsGeoJSON(%s,%s,%s,%s)' %(second['version'],
2679 self.expand(first), second['precision'], second['options'])
2680
2681 - def ST_ASTEXT(self, first):
2682 """
2683 http://postgis.org/docs/ST_AsText.html
2684 """
2685 return 'ST_AsText(%s)' %(self.expand(first))
2686
2687 - def ST_X(self, first):
2688 """
2689 http://postgis.org/docs/ST_X.html
2690 """
2691 return 'ST_X(%s)' %(self.expand(first))
2692
2693 - def ST_Y(self, first):
2694 """
2695 http://postgis.org/docs/ST_Y.html
2696 """
2697 return 'ST_Y(%s)' %(self.expand(first))
2698
2699 - def ST_CONTAINS(self, first, second):
2700 """
2701 http://postgis.org/docs/ST_Contains.html
2702 """
2703 return 'ST_Contains(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2704
2705 - def ST_DISTANCE(self, first, second):
2706 """
2707 http://postgis.org/docs/ST_Distance.html
2708 """
2709 return 'ST_Distance(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2710
2711 - def ST_EQUALS(self, first, second):
2712 """
2713 http://postgis.org/docs/ST_Equals.html
2714 """
2715 return 'ST_Equals(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2716
2717 - def ST_INTERSECTS(self, first, second):
2718 """
2719 http://postgis.org/docs/ST_Intersects.html
2720 """
2721 return 'ST_Intersects(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2722
2723 - def ST_OVERLAPS(self, first, second):
2724 """
2725 http://postgis.org/docs/ST_Overlaps.html
2726 """
2727 return 'ST_Overlaps(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2728
2729 - def ST_SIMPLIFY(self, first, second):
2730 """
2731 http://postgis.org/docs/ST_Simplify.html
2732 """
2733 return 'ST_Simplify(%s,%s)' %(self.expand(first), self.expand(second, 'double'))
2734
2735 - def ST_TOUCHES(self, first, second):
2736 """
2737 http://postgis.org/docs/ST_Touches.html
2738 """
2739 return 'ST_Touches(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2740
2741 - def ST_WITHIN(self, first, second):
2742 """
2743 http://postgis.org/docs/ST_Within.html
2744 """
2745 return 'ST_Within(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2746
2747 - def represent(self, obj, fieldtype):
2748 field_is_type = fieldtype.startswith
2749 if field_is_type('geo'):
2750 srid = 4326
2751 geotype, parms = fieldtype[:-1].split('(')
2752 parms = parms.split(',')
2753 if len(parms) >= 2:
2754 schema, srid = parms[:2]
2755 if field_is_type('geometry'):
2756 value = "ST_GeomFromText('%s',%s)" %(obj, srid)
2757 elif field_is_type('geography'):
2758 value = "ST_GeogFromText('SRID=%s;%s')" %(srid, obj)
2759
2760
2761 return value
2762 return BaseAdapter.represent(self, obj, fieldtype)
2763
2764 -class NewPostgreSQLAdapter(PostgreSQLAdapter):
2765 drivers = ('psycopg2','pg8000')
2766
2767 types = {
2768 'boolean': 'CHAR(1)',
2769 'string': 'VARCHAR(%(length)s)',
2770 'text': 'TEXT',
2771 'json': 'TEXT',
2772 'password': 'VARCHAR(%(length)s)',
2773 'blob': 'BYTEA',
2774 'upload': 'VARCHAR(%(length)s)',
2775 'integer': 'INTEGER',
2776 'bigint': 'BIGINT',
2777 'float': 'FLOAT',
2778 'double': 'FLOAT8',
2779 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2780 'date': 'DATE',
2781 'time': 'TIME',
2782 'datetime': 'TIMESTAMP',
2783 'id': 'SERIAL PRIMARY KEY',
2784 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2785 'list:integer': 'BIGINT[]',
2786 'list:string': 'TEXT[]',
2787 'list:reference': 'BIGINT[]',
2788 'geometry': 'GEOMETRY',
2789 'geography': 'GEOGRAPHY',
2790 'big-id': 'BIGSERIAL PRIMARY KEY',
2791 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2792 }
2793
2794 - def parse_list_integers(self, value, field_type):
2796
2797 - def parse_list_references(self, value, field_type):
2798 return [self.parse_reference(r, field_type[5:]) for r in value]
2799
2800 - def parse_list_strings(self, value, field_type):
2802
2803 - def represent(self, obj, fieldtype):
2804 field_is_type = fieldtype.startswith
2805 if field_is_type('list:'):
2806 if not obj:
2807 obj = []
2808 elif not isinstance(obj, (list, tuple)):
2809 obj = [obj]
2810 if field_is_type('list:string'):
2811 obj = map(str,obj)
2812 else:
2813 obj = map(int,obj)
2814 return 'ARRAY[%s]' % ','.join(repr(item) for item in obj)
2815 return BaseAdapter.represent(self, obj, fieldtype)
2816
2817
2818 -class JDBCPostgreSQLAdapter(PostgreSQLAdapter):
2819 drivers = ('zxJDBC',)
2820
2821 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+)$')
2822
2823 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
2824 credential_decoder=IDENTITY, driver_args={},
2825 adapter_args={}, do_connect=True, after_connection=None ):
2826 self.db = db
2827 self.dbengine = "postgres"
2828 self.uri = uri
2829 if do_connect: self.find_driver(adapter_args,uri)
2830 self.pool_size = pool_size
2831 self.folder = folder
2832 self.db_codec = db_codec
2833 self._after_connection = after_connection
2834 self.find_or_make_work_folder()
2835 ruri = uri.split('://',1)[1]
2836 m = self.REGEX_URI.match(ruri)
2837 if not m:
2838 raise SyntaxError("Invalid URI string in DAL")
2839 user = credential_decoder(m.group('user'))
2840 if not user:
2841 raise SyntaxError('User required')
2842 password = credential_decoder(m.group('password'))
2843 if not password:
2844 password = ''
2845 host = m.group('host')
2846 if not host:
2847 raise SyntaxError('Host name required')
2848 db = m.group('db')
2849 if not db:
2850 raise SyntaxError('Database name required')
2851 port = m.group('port') or '5432'
2852 msg = ('jdbc:postgresql://%s:%s/%s' % (host, port, db), user, password)
2853 def connector(msg=msg,driver_args=driver_args):
2854 return self.driver.connect(*msg,**driver_args)
2855 self.connector = connector
2856 if do_connect: self.reconnect()
2857
2858 - def after_connection(self):
2859 self.connection.set_client_encoding('UTF8')
2860 self.execute('BEGIN;')
2861 self.execute("SET CLIENT_ENCODING TO 'UNICODE';")
2862 self.try_json()
2863
2866 drivers = ('cx_Oracle',)
2867
2868 commit_on_alter_table = False
2869 types = {
2870 'boolean': 'CHAR(1)',
2871 'string': 'VARCHAR2(%(length)s)',
2872 'text': 'CLOB',
2873 'json': 'CLOB',
2874 'password': 'VARCHAR2(%(length)s)',
2875 'blob': 'CLOB',
2876 'upload': 'VARCHAR2(%(length)s)',
2877 'integer': 'INT',
2878 'bigint': 'NUMBER',
2879 'float': 'FLOAT',
2880 'double': 'BINARY_DOUBLE',
2881 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2882 'date': 'DATE',
2883 'time': 'CHAR(8)',
2884 'datetime': 'DATE',
2885 'id': 'NUMBER PRIMARY KEY',
2886 'reference': 'NUMBER, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2887 'list:integer': 'CLOB',
2888 'list:string': 'CLOB',
2889 'list:reference': 'CLOB',
2890 'big-id': 'NUMBER PRIMARY KEY',
2891 'big-reference': 'NUMBER, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2892 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2893 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
2894 }
2895
2897 return '%s_sequence' % tablename
2898
2900 return '%s_trigger' % tablename
2901
2903 return 'LEFT OUTER JOIN'
2904
2906 return 'dbms_random.value'
2907
2908 - def NOT_NULL(self,default,field_type):
2909 return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
2910
2911 - def _drop(self,table,mode):
2914
2915 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
2916 if limitby:
2917 (lmin, lmax) = limitby
2918 if len(sql_w) > 1:
2919 sql_w_row = sql_w + ' AND w_row > %i' % lmin
2920 else:
2921 sql_w_row = 'WHERE w_row > %i' % lmin
2922 return 'SELECT %s %s FROM (SELECT w_tmp.*, ROWNUM w_row FROM (SELECT %s FROM %s%s%s) w_tmp WHERE ROWNUM<=%i) %s %s %s;' % (sql_s, sql_f, sql_f, sql_t, sql_w, sql_o, lmax, sql_t, sql_w_row, sql_o)
2923 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
2924
2930
2932 if fieldtype == 'blob':
2933 obj = base64.b64encode(str(obj))
2934 return ":CLOB('%s')" % obj
2935 elif fieldtype == 'date':
2936 if isinstance(obj, (datetime.date, datetime.datetime)):
2937 obj = obj.isoformat()[:10]
2938 else:
2939 obj = str(obj)
2940 return "to_date('%s','yyyy-mm-dd')" % obj
2941 elif fieldtype == 'datetime':
2942 if isinstance(obj, datetime.datetime):
2943 obj = obj.isoformat()[:19].replace('T',' ')
2944 elif isinstance(obj, datetime.date):
2945 obj = obj.isoformat()[:10]+' 00:00:00'
2946 else:
2947 obj = str(obj)
2948 return "to_date('%s','yyyy-mm-dd hh24:mi:ss')" % obj
2949 return None
2950
2951 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
2952 credential_decoder=IDENTITY, driver_args={},
2953 adapter_args={}, do_connect=True, after_connection=None):
2954 self.db = db
2955 self.dbengine = "oracle"
2956 self.uri = uri
2957 if do_connect: self.find_driver(adapter_args,uri)
2958 self.pool_size = pool_size
2959 self.folder = folder
2960 self.db_codec = db_codec
2961 self._after_connection = after_connection
2962 self.find_or_make_work_folder()
2963 ruri = uri.split('://',1)[1]
2964 if not 'threaded' in driver_args:
2965 driver_args['threaded']=True
2966 def connector(uri=ruri,driver_args=driver_args):
2967 return self.driver.connect(uri,**driver_args)
2968 self.connector = connector
2969 if do_connect: self.reconnect()
2970
2972 self.execute("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD HH24:MI:SS';")
2973 self.execute("ALTER SESSION SET NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS';")
2974
2975 oracle_fix = re.compile("[^']*('[^']*'[^']*)*\:(?P<clob>CLOB\('([^']+|'')*'\))")
2976
2977 - def execute(self, command, args=None):
2978 args = args or []
2979 i = 1
2980 while True:
2981 m = self.oracle_fix.match(command)
2982 if not m:
2983 break
2984 command = command[:m.start('clob')] + str(i) + command[m.end('clob'):]
2985 args.append(m.group('clob')[6:-2].replace("''", "'"))
2986 i += 1
2987 if command[-1:]==';':
2988 command = command[:-1]
2989 return self.log_execute(command, args)
2990
2992 tablename = table._tablename
2993 sequence_name = table._sequence_name
2994 trigger_name = table._trigger_name
2995 self.execute(query)
2996 self.execute('CREATE SEQUENCE %s START WITH 1 INCREMENT BY 1 NOMAXVALUE MINVALUE -1;' % sequence_name)
2997 self.execute("""
2998 CREATE OR REPLACE TRIGGER %(trigger_name)s BEFORE INSERT ON %(tablename)s FOR EACH ROW
2999 DECLARE
3000 curr_val NUMBER;
3001 diff_val NUMBER;
3002 PRAGMA autonomous_transaction;
3003 BEGIN
3004 IF :NEW.id IS NOT NULL THEN
3005 EXECUTE IMMEDIATE 'SELECT %(sequence_name)s.nextval FROM dual' INTO curr_val;
3006 diff_val := :NEW.id - curr_val - 1;
3007 IF diff_val != 0 THEN
3008 EXECUTE IMMEDIATE 'alter sequence %(sequence_name)s increment by '|| diff_val;
3009 EXECUTE IMMEDIATE 'SELECT %(sequence_name)s.nextval FROM dual' INTO curr_val;
3010 EXECUTE IMMEDIATE 'alter sequence %(sequence_name)s increment by 1';
3011 END IF;
3012 END IF;
3013 SELECT %(sequence_name)s.nextval INTO :NEW.id FROM DUAL;
3014 END;
3015 """ % dict(trigger_name=trigger_name, tablename=tablename, sequence_name=sequence_name))
3016
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3032 if any(x[1]==cx_Oracle.CLOB for x in self.cursor.description):
3033 return [tuple([(c.read() if type(c) == cx_Oracle.LOB else c) \
3034 for c in r]) for r in self.cursor]
3035 else:
3036 return self.cursor.fetchall()
3037
3039 drivers = ('pyodbc',)
3040 T_SEP = 'T'
3041
3042 types = {
3043 'boolean': 'BIT',
3044 'string': 'VARCHAR(%(length)s)',
3045 'text': 'TEXT',
3046 'json': 'TEXT',
3047 'password': 'VARCHAR(%(length)s)',
3048 'blob': 'IMAGE',
3049 'upload': 'VARCHAR(%(length)s)',
3050 'integer': 'INT',
3051 'bigint': 'BIGINT',
3052 'float': 'FLOAT',
3053 'double': 'FLOAT',
3054 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3055 'date': 'DATETIME',
3056 'time': 'CHAR(8)',
3057 'datetime': 'DATETIME',
3058 'id': 'INT IDENTITY PRIMARY KEY',
3059 'reference': 'INT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3060 'list:integer': 'TEXT',
3061 'list:string': 'TEXT',
3062 'list:reference': 'TEXT',
3063 'geometry': 'geometry',
3064 'geography': 'geography',
3065 'big-id': 'BIGINT IDENTITY PRIMARY KEY',
3066 'big-reference': 'BIGINT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3067 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3068 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3069 }
3070
3072 return '; ALTER TABLE %s ADD ' % tablename
3073
3076
3078 return "DATEPART(%s,%s)" % (what, self.expand(field))
3079
3081 return 'LEFT OUTER JOIN'
3082
3085
3088
3090 return 'SUBSTRING(%s,%s,%s)' % (self.expand(field), parameters[0], parameters[1])
3091
3093 return 'PRIMARY KEY CLUSTERED (%s)' % key
3094
3096 if what == 'LENGTH':
3097 what = 'LEN'
3098 return "%s(%s)" % (what, self.expand(first))
3099
3100
3101 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3102 if limitby:
3103 (lmin, lmax) = limitby
3104 sql_s += ' TOP %i' % lmax
3105 if 'GROUP BY' in sql_o:
3106 sql_o = sql_o[:sql_o.find('ORDER BY ')]
3107 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3108
3109 TRUE = 1
3110 FALSE = 0
3111
3112 REGEX_DSN = re.compile('^(?P<dsn>.+)$')
3113 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?(?P<urlargs>.*))?$')
3114 REGEX_ARGPATTERN = re.compile('(?P<argkey>[^=]+)=(?P<argvalue>[^&]*)')
3115
3116 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3117 credential_decoder=IDENTITY, driver_args={},
3118 adapter_args={}, do_connect=True, srid=4326,
3119 after_connection=None):
3120 self.db = db
3121 self.dbengine = "mssql"
3122 self.uri = uri
3123 if do_connect: self.find_driver(adapter_args,uri)
3124 self.pool_size = pool_size
3125 self.folder = folder
3126 self.db_codec = db_codec
3127 self._after_connection = after_connection
3128 self.srid = srid
3129 self.find_or_make_work_folder()
3130
3131 ruri = uri.split('://',1)[1]
3132 if '@' not in ruri:
3133 try:
3134 m = self.REGEX_DSN.match(ruri)
3135 if not m:
3136 raise SyntaxError(
3137 'Parsing uri string(%s) has no result' % self.uri)
3138 dsn = m.group('dsn')
3139 if not dsn:
3140 raise SyntaxError('DSN required')
3141 except SyntaxError:
3142 e = sys.exc_info()[1]
3143 LOGGER.error('NdGpatch error')
3144 raise e
3145
3146 cnxn = dsn
3147 else:
3148 m = self.REGEX_URI.match(ruri)
3149 if not m:
3150 raise SyntaxError(
3151 "Invalid URI string in DAL: %s" % self.uri)
3152 user = credential_decoder(m.group('user'))
3153 if not user:
3154 raise SyntaxError('User required')
3155 password = credential_decoder(m.group('password'))
3156 if not password:
3157 password = ''
3158 host = m.group('host')
3159 if not host:
3160 raise SyntaxError('Host name required')
3161 db = m.group('db')
3162 if not db:
3163 raise SyntaxError('Database name required')
3164 port = m.group('port') or '1433'
3165
3166
3167
3168 argsdict = { 'DRIVER':'{SQL Server}' }
3169 urlargs = m.group('urlargs') or ''
3170 for argmatch in self.REGEX_ARGPATTERN.finditer(urlargs):
3171 argsdict[str(argmatch.group('argkey')).upper()] = argmatch.group('argvalue')
3172 urlargs = ';'.join(['%s=%s' % (ak, av) for (ak, av) in argsdict.iteritems()])
3173 cnxn = 'SERVER=%s;PORT=%s;DATABASE=%s;UID=%s;PWD=%s;%s' \
3174 % (host, port, db, user, password, urlargs)
3175 def connector(cnxn=cnxn,driver_args=driver_args):
3176 return self.driver.connect(cnxn,**driver_args)
3177 self.connector = connector
3178 if do_connect: self.reconnect()
3179
3181
3182 self.execute('SELECT SCOPE_IDENTITY();')
3183 return int(self.cursor.fetchone()[0])
3184
3186 return pyodbc.IntegrityError
3187
3188 - def rowslice(self,rows,minimum=0,maximum=None):
3189 if maximum is None:
3190 return rows[minimum:]
3191 return rows[minimum:maximum]
3192
3193 - def EPOCH(self, first):
3194 return "DATEDIFF(second, '1970-01-01 00:00:00', %s)" % self.expand(first)
3195
3196
3197
3198
3199
3200 - def ST_ASTEXT(self, first):
3201 return '%s.STAsText()' %(self.expand(first))
3202
3205
3208
3211
3214
3217
3218
3219
3222
3225
3227 field_is_type = fieldtype.startswith
3228 if field_is_type('geometry'):
3229 srid = 0
3230 geotype, parms = fieldtype[:-1].split('(')
3231 if parms:
3232 srid = parms
3233 return "geometry::STGeomFromText('%s',%s)" %(obj, srid)
3234 elif fieldtype == 'geography':
3235 srid = 4326
3236 geotype, parms = fieldtype[:-1].split('(')
3237 if parms:
3238 srid = parms
3239 return "geography::STGeomFromText('%s',%s)" %(obj, srid)
3240
3241
3242 return "geometry::STGeomFromText('%s',%s)" %(obj, srid)
3243 return BaseAdapter.represent(self, obj, fieldtype)
3244
3247 """ experimental support for pagination in MSSQL"""
3248 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3249 if limitby:
3250 (lmin, lmax) = limitby
3251 if lmin == 0:
3252 sql_s += ' TOP %i' % lmax
3253 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3254 lmin += 1
3255 sql_o_inner = sql_o[sql_o.find('ORDER BY ')+9:]
3256 sql_g_inner = sql_o[:sql_o.find('ORDER BY ')]
3257 sql_f_outer = ['f_%s' % f for f in range(len(sql_f.split(',')))]
3258 sql_f_inner = [f for f in sql_f.split(',')]
3259 sql_f_iproxy = ['%s AS %s' % (o, n) for (o, n) in zip(sql_f_inner, sql_f_outer)]
3260 sql_f_iproxy = ', '.join(sql_f_iproxy)
3261 sql_f_oproxy = ', '.join(sql_f_outer)
3262 return 'SELECT %s %s FROM (SELECT %s ROW_NUMBER() OVER (ORDER BY %s) AS w_row, %s FROM %s%s%s) TMP WHERE w_row BETWEEN %i AND %s;' % (sql_s,sql_f_oproxy,sql_s,sql_f,sql_f_iproxy,sql_t,sql_w,sql_g_inner,lmin,lmax)
3263 return 'SELECT %s %s FROM %s%s%s;' % (sql_s,sql_f,sql_t,sql_w,sql_o)
3264 - def rowslice(self,rows,minimum=0,maximum=None):
3266
3269 drivers = ('pyodbc',)
3270
3271 types = {
3272 'boolean': 'CHAR(1)',
3273 'string': 'NVARCHAR(%(length)s)',
3274 'text': 'NTEXT',
3275 'json': 'NTEXT',
3276 'password': 'NVARCHAR(%(length)s)',
3277 'blob': 'IMAGE',
3278 'upload': 'NVARCHAR(%(length)s)',
3279 'integer': 'INT',
3280 'bigint': 'BIGINT',
3281 'float': 'FLOAT',
3282 'double': 'FLOAT',
3283 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3284 'date': 'DATETIME',
3285 'time': 'CHAR(8)',
3286 'datetime': 'DATETIME',
3287 'id': 'INT IDENTITY PRIMARY KEY',
3288 'reference': 'INT, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3289 'list:integer': 'NTEXT',
3290 'list:string': 'NTEXT',
3291 'list:reference': 'NTEXT',
3292 'big-id': 'BIGINT IDENTITY PRIMARY KEY',
3293 'big-reference': 'BIGINT, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3294 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3295 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3296 }
3297
3299 value = BaseAdapter.represent(self, obj, fieldtype)
3300 if fieldtype in ('string','text', 'json') and value[:1]=="'":
3301 value = 'N'+value
3302 return value
3303
3306
3309 drivers = ('Sybase',)
3310
3311 types = {
3312 'boolean': 'BIT',
3313 'string': 'CHAR VARYING(%(length)s)',
3314 'text': 'TEXT',
3315 'json': 'TEXT',
3316 'password': 'CHAR VARYING(%(length)s)',
3317 'blob': 'IMAGE',
3318 'upload': 'CHAR VARYING(%(length)s)',
3319 'integer': 'INT',
3320 'bigint': 'BIGINT',
3321 'float': 'FLOAT',
3322 'double': 'FLOAT',
3323 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3324 'date': 'DATETIME',
3325 'time': 'CHAR(8)',
3326 'datetime': 'DATETIME',
3327 'id': 'INT IDENTITY PRIMARY KEY',
3328 'reference': 'INT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3329 'list:integer': 'TEXT',
3330 'list:string': 'TEXT',
3331 'list:reference': 'TEXT',
3332 'geometry': 'geometry',
3333 'geography': 'geography',
3334 'big-id': 'BIGINT IDENTITY PRIMARY KEY',
3335 'big-reference': 'BIGINT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3336 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3337 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3338 }
3339
3340
3341 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3342 credential_decoder=IDENTITY, driver_args={},
3343 adapter_args={}, do_connect=True, srid=4326,
3344 after_connection=None):
3345 self.db = db
3346 self.dbengine = "sybase"
3347 self.uri = uri
3348 if do_connect: self.find_driver(adapter_args,uri)
3349 self.pool_size = pool_size
3350 self.folder = folder
3351 self.db_codec = db_codec
3352 self._after_connection = after_connection
3353 self.srid = srid
3354 self.find_or_make_work_folder()
3355
3356 ruri = uri.split('://',1)[1]
3357 if '@' not in ruri:
3358 try:
3359 m = self.REGEX_DSN.match(ruri)
3360 if not m:
3361 raise SyntaxError(
3362 'Parsing uri string(%s) has no result' % self.uri)
3363 dsn = m.group('dsn')
3364 if not dsn:
3365 raise SyntaxError('DSN required')
3366 except SyntaxError:
3367 e = sys.exc_info()[1]
3368 LOGGER.error('NdGpatch error')
3369 raise e
3370 else:
3371 m = self.REGEX_URI.match(uri)
3372 if not m:
3373 raise SyntaxError(
3374 "Invalid URI string in DAL: %s" % self.uri)
3375 user = credential_decoder(m.group('user'))
3376 if not user:
3377 raise SyntaxError('User required')
3378 password = credential_decoder(m.group('password'))
3379 if not password:
3380 password = ''
3381 host = m.group('host')
3382 if not host:
3383 raise SyntaxError('Host name required')
3384 db = m.group('db')
3385 if not db:
3386 raise SyntaxError('Database name required')
3387 port = m.group('port') or '1433'
3388
3389 dsn = 'sybase:host=%s:%s;dbname=%s' % (host,port,db)
3390
3391 driver_args.update(user = credential_decoder(user),
3392 password = credential_decoder(password))
3393
3394 def connector(dsn=dsn,driver_args=driver_args):
3395 return self.driver.connect(dsn,**driver_args)
3396 self.connector = connector
3397 if do_connect: self.reconnect()
3398
3401
3404 drivers = ('kinterbasdb','firebirdsql','fdb','pyodbc')
3405
3406 commit_on_alter_table = False
3407 support_distributed_transaction = True
3408 types = {
3409 'boolean': 'CHAR(1)',
3410 'string': 'VARCHAR(%(length)s)',
3411 'text': 'BLOB SUB_TYPE 1',
3412 'json': 'BLOB SUB_TYPE 1',
3413 'password': 'VARCHAR(%(length)s)',
3414 'blob': 'BLOB SUB_TYPE 0',
3415 'upload': 'VARCHAR(%(length)s)',
3416 'integer': 'INTEGER',
3417 'bigint': 'BIGINT',
3418 'float': 'FLOAT',
3419 'double': 'DOUBLE PRECISION',
3420 'decimal': 'DECIMAL(%(precision)s,%(scale)s)',
3421 'date': 'DATE',
3422 'time': 'TIME',
3423 'datetime': 'TIMESTAMP',
3424 'id': 'INTEGER PRIMARY KEY',
3425 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3426 'list:integer': 'BLOB SUB_TYPE 1',
3427 'list:string': 'BLOB SUB_TYPE 1',
3428 'list:reference': 'BLOB SUB_TYPE 1',
3429 'big-id': 'BIGINT PRIMARY KEY',
3430 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3431 }
3432
3434 return 'genid_%s' % tablename
3435
3437 return 'trg_id_%s' % tablename
3438
3441
3442 - def EPOCH(self, first):
3443 return "DATEDIFF(second, '1970-01-01 00:00:00', %s)" % self.expand(first)
3444
3445 - def NOT_NULL(self,default,field_type):
3446 return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
3447
3449 return 'SUBSTRING(%s from %s for %s)' % (self.expand(field), parameters[0], parameters[1])
3450
3452 "case in-sensitive like operator"
3453 return '(%s CONTAINING %s)' % (self.expand(first),
3454 self.expand(second, 'string'))
3455
3456 - def CONTAINS(self, first, second, case_sensitive=False):
3462
3463 - def _drop(self,table,mode):
3466
3467 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3468 if limitby:
3469 (lmin, lmax) = limitby
3470 sql_s = ' FIRST %i SKIP %i %s' % (lmax - lmin, lmin, sql_s)
3471 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3472
3474 return ['DELETE FROM %s;' % table._tablename,
3475 'SET GENERATOR %s TO 0;' % table._sequence_name]
3476
3477 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+?)(\?set_encoding=(?P<charset>\w+))?$')
3478
3479 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3480 credential_decoder=IDENTITY, driver_args={},
3481 adapter_args={}, do_connect=True, after_connection=None):
3482 self.db = db
3483 self.dbengine = "firebird"
3484 self.uri = uri
3485 if do_connect: self.find_driver(adapter_args,uri)
3486 self.pool_size = pool_size
3487 self.folder = folder
3488 self.db_codec = db_codec
3489 self._after_connection = after_connection
3490 self.find_or_make_work_folder()
3491 ruri = uri.split('://',1)[1]
3492 m = self.REGEX_URI.match(ruri)
3493 if not m:
3494 raise SyntaxError("Invalid URI string in DAL: %s" % self.uri)
3495 user = credential_decoder(m.group('user'))
3496 if not user:
3497 raise SyntaxError('User required')
3498 password = credential_decoder(m.group('password'))
3499 if not password:
3500 password = ''
3501 host = m.group('host')
3502 if not host:
3503 raise SyntaxError('Host name required')
3504 port = int(m.group('port') or 3050)
3505 db = m.group('db')
3506 if not db:
3507 raise SyntaxError('Database name required')
3508 charset = m.group('charset') or 'UTF8'
3509 driver_args.update(dsn='%s/%s:%s' % (host,port,db),
3510 user = credential_decoder(user),
3511 password = credential_decoder(password),
3512 charset = charset)
3513
3514 def connector(driver_args=driver_args):
3515 return self.driver.connect(**driver_args)
3516 self.connector = connector
3517 if do_connect: self.reconnect()
3518
3527
3532
3535 drivers = ('kinterbasdb','firebirdsql','fdb','pyodbc')
3536
3537 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<path>[^\?]+)(\?set_encoding=(?P<charset>\w+))?$')
3538
3539 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3540 credential_decoder=IDENTITY, driver_args={},
3541 adapter_args={}, do_connect=True, after_connection=None):
3542 self.db = db
3543 self.dbengine = "firebird"
3544 self.uri = uri
3545 if do_connect: self.find_driver(adapter_args,uri)
3546 self.pool_size = pool_size
3547 self.folder = folder
3548 self.db_codec = db_codec
3549 self._after_connection = after_connection
3550 self.find_or_make_work_folder()
3551 ruri = uri.split('://',1)[1]
3552 m = self.REGEX_URI.match(ruri)
3553 if not m:
3554 raise SyntaxError(
3555 "Invalid URI string in DAL: %s" % self.uri)
3556 user = credential_decoder(m.group('user'))
3557 if not user:
3558 raise SyntaxError('User required')
3559 password = credential_decoder(m.group('password'))
3560 if not password:
3561 password = ''
3562 pathdb = m.group('path')
3563 if not pathdb:
3564 raise SyntaxError('Path required')
3565 charset = m.group('charset')
3566 if not charset:
3567 charset = 'UTF8'
3568 host = ''
3569 driver_args.update(host=host,
3570 database=pathdb,
3571 user=credential_decoder(user),
3572 password=credential_decoder(password),
3573 charset=charset)
3574
3575 def connector(driver_args=driver_args):
3576 return self.driver.connect(**driver_args)
3577 self.connector = connector
3578 if do_connect: self.reconnect()
3579
3685
3690
3693
3696
3708
3710 drivers = ('pyodbc',)
3711
3712 types = {
3713 'boolean': 'CHAR(1)',
3714 'string': 'VARCHAR(%(length)s)',
3715 'text': 'CLOB',
3716 'json': 'CLOB',
3717 'password': 'VARCHAR(%(length)s)',
3718 'blob': 'BLOB',
3719 'upload': 'VARCHAR(%(length)s)',
3720 'integer': 'INT',
3721 'bigint': 'BIGINT',
3722 'float': 'REAL',
3723 'double': 'DOUBLE',
3724 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3725 'date': 'DATE',
3726 'time': 'TIME',
3727 'datetime': 'TIMESTAMP',
3728 'id': 'INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL',
3729 'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3730 'list:integer': 'CLOB',
3731 'list:string': 'CLOB',
3732 'list:reference': 'CLOB',
3733 'big-id': 'BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL',
3734 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3735 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3736 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3737 }
3738
3740 return 'LEFT OUTER JOIN'
3741
3744
3745 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3746 if limitby:
3747 (lmin, lmax) = limitby
3748 sql_o += ' FETCH FIRST %i ROWS ONLY' % lmax
3749 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3750
3752 if fieldtype == 'blob':
3753 obj = base64.b64encode(str(obj))
3754 return "BLOB('%s')" % obj
3755 elif fieldtype == 'datetime':
3756 if isinstance(obj, datetime.datetime):
3757 obj = obj.isoformat()[:19].replace('T','-').replace(':','.')
3758 elif isinstance(obj, datetime.date):
3759 obj = obj.isoformat()[:10]+'-00.00.00'
3760 return "'%s'" % obj
3761 return None
3762
3763 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3764 credential_decoder=IDENTITY, driver_args={},
3765 adapter_args={}, do_connect=True, after_connection=None):
3766 self.db = db
3767 self.dbengine = "db2"
3768 self.uri = uri
3769 if do_connect: self.find_driver(adapter_args,uri)
3770 self.pool_size = pool_size
3771 self.folder = folder
3772 self.db_codec = db_codec
3773 self._after_connection = after_connection
3774 self.find_or_make_work_folder()
3775 ruri = uri.split('://', 1)[1]
3776 def connector(cnxn=ruri,driver_args=driver_args):
3777 return self.driver.connect(cnxn,**driver_args)
3778 self.connector = connector
3779 if do_connect: self.reconnect()
3780
3782 if command[-1:]==';':
3783 command = command[:-1]
3784 return self.log_execute(command)
3785
3787 self.execute('SELECT DISTINCT IDENTITY_VAL_LOCAL() FROM %s;' % table)
3788 return int(self.cursor.fetchone()[0])
3789
3790 - def rowslice(self,rows,minimum=0,maximum=None):
3791 if maximum is None:
3792 return rows[minimum:]
3793 return rows[minimum:maximum]
3794
3797 drivers = ('pyodbc',)
3798
3799 types = {
3800 'boolean': 'CHAR(1)',
3801 'string': 'VARCHAR(%(length)s)',
3802 'text': 'CLOB',
3803 'json': 'CLOB',
3804 'password': 'VARCHAR(%(length)s)',
3805 'blob': 'BLOB',
3806 'upload': 'VARCHAR(%(length)s)',
3807 'integer': 'INT',
3808 'bigint': 'BIGINT',
3809 'float': 'REAL',
3810 'double': 'DOUBLE',
3811 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3812 'date': 'DATE',
3813 'time': 'TIME',
3814 'datetime': 'TIMESTAMP',
3815
3816
3817 'id': 'INT GENERATED ALWAYS AS IDENTITY',
3818 'reference': 'INT',
3819 'list:integer': 'CLOB',
3820 'list:string': 'CLOB',
3821 'list:reference': 'CLOB',
3822 'big-id': 'BIGINT GENERATED ALWAYS AS IDENTITY',
3823 'big-reference': 'BIGINT',
3824 'reference FK': ' REFERENCES %(foreign_key)s',
3825 'reference TFK': ' FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s)',
3826 }
3827
3828 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3829 credential_decoder=IDENTITY, driver_args={},
3830 adapter_args={}, do_connect=True, after_connection=None):
3831 self.db = db
3832 self.dbengine = "teradata"
3833 self.uri = uri
3834 if do_connect: self.find_driver(adapter_args,uri)
3835 self.pool_size = pool_size
3836 self.folder = folder
3837 self.db_codec = db_codec
3838 self._after_connection = after_connection
3839 self.find_or_make_work_folder()
3840 ruri = uri.split('://', 1)[1]
3841 def connector(cnxn=ruri,driver_args=driver_args):
3842 return self.driver.connect(cnxn,**driver_args)
3843 self.connector = connector
3844 if do_connect: self.reconnect()
3845
3847 return 'LEFT OUTER JOIN'
3848
3849
3850 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3851 if limitby:
3852 (lmin, lmax) = limitby
3853 sql_s += ' TOP %i' % lmax
3854 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3855
3857 tablename = table._tablename
3858 return ['DELETE FROM %s ALL;' % (tablename)]
3859
3860 INGRES_SEQNAME='ii***lineitemsequence'
3865 drivers = ('pyodbc',)
3866
3867 types = {
3868 'boolean': 'CHAR(1)',
3869 'string': 'VARCHAR(%(length)s)',
3870 'text': 'CLOB',
3871 'json': 'CLOB',
3872 'password': 'VARCHAR(%(length)s)',
3873 'blob': 'BLOB',
3874 'upload': 'VARCHAR(%(length)s)',
3875 'integer': 'INTEGER4',
3876 'bigint': 'BIGINT',
3877 'float': 'FLOAT',
3878 'double': 'FLOAT8',
3879 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3880 'date': 'ANSIDATE',
3881 'time': 'TIME WITHOUT TIME ZONE',
3882 'datetime': 'TIMESTAMP WITHOUT TIME ZONE',
3883 'id': 'int not null unique with default next value for %s' % INGRES_SEQNAME,
3884 'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3885 'list:integer': 'CLOB',
3886 'list:string': 'CLOB',
3887 'list:reference': 'CLOB',
3888 'big-id': 'bigint not null unique with default next value for %s' % INGRES_SEQNAME,
3889 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3890 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3891 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3892 }
3893
3895 return 'LEFT OUTER JOIN'
3896
3899
3900 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3901 if limitby:
3902 (lmin, lmax) = limitby
3903 fetch_amt = lmax - lmin
3904 if fetch_amt:
3905 sql_s += ' FIRST %d ' % (fetch_amt, )
3906 if lmin:
3907
3908 sql_o += ' OFFSET %d' % (lmin, )
3909 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3910
3911 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3912 credential_decoder=IDENTITY, driver_args={},
3913 adapter_args={}, do_connect=True, after_connection=None):
3914 self.db = db
3915 self.dbengine = "ingres"
3916 self._driver = pyodbc
3917 self.uri = uri
3918 if do_connect: self.find_driver(adapter_args,uri)
3919 self.pool_size = pool_size
3920 self.folder = folder
3921 self.db_codec = db_codec
3922 self._after_connection = after_connection
3923 self.find_or_make_work_folder()
3924 connstr = uri.split(':', 1)[1]
3925
3926 connstr = connstr.lstrip()
3927 while connstr.startswith('/'):
3928 connstr = connstr[1:]
3929 if '=' in connstr:
3930
3931 ruri = connstr
3932 else:
3933
3934 database_name = connstr
3935 default_driver_name = 'Ingres'
3936 vnode = '(local)'
3937 servertype = 'ingres'
3938 ruri = 'Driver={%s};Server=%s;Database=%s' % (default_driver_name, vnode, database_name)
3939 def connector(cnxn=ruri,driver_args=driver_args):
3940 return self.driver.connect(cnxn,**driver_args)
3941
3942 self.connector = connector
3943
3944
3945 if do_connect: self.reconnect()
3946
3948
3949
3950
3951 if hasattr(table,'_primarykey'):
3952 modify_tbl_sql = 'modify %s to btree unique on %s' % \
3953 (table._tablename,
3954 ', '.join(["'%s'" % x for x in table.primarykey]))
3955 self.execute(modify_tbl_sql)
3956 else:
3957 tmp_seqname='%s_iisq' % table._tablename
3958 query=query.replace(INGRES_SEQNAME, tmp_seqname)
3959 self.execute('create sequence %s' % tmp_seqname)
3960 self.execute(query)
3961 self.execute('modify %s to btree unique on %s' % (table._tablename, 'id'))
3962
3963
3965 tmp_seqname='%s_iisq' % table
3966 self.execute('select current value for %s' % tmp_seqname)
3967 return int(self.cursor.fetchone()[0])
3968
3970 return self._driver.IntegrityError
3971
3974
3975 drivers = ('pyodbc',)
3976
3977 types = {
3978 'boolean': 'CHAR(1)',
3979 'string': 'NVARCHAR(%(length)s)',
3980 'text': 'NCLOB',
3981 'json': 'NCLOB',
3982 'password': 'NVARCHAR(%(length)s)',
3983 'blob': 'BLOB',
3984 'upload': 'VARCHAR(%(length)s)',
3985 'integer': 'INTEGER4',
3986 'bigint': 'BIGINT',
3987 'float': 'FLOAT',
3988 'double': 'FLOAT8',
3989 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3990 'date': 'ANSIDATE',
3991 'time': 'TIME WITHOUT TIME ZONE',
3992 'datetime': 'TIMESTAMP WITHOUT TIME ZONE',
3993 'id': 'INTEGER4 not null unique with default next value for %s'% INGRES_SEQNAME,
3994 'reference': 'INTEGER4, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3995 'list:integer': 'NCLOB',
3996 'list:string': 'NCLOB',
3997 'list:reference': 'NCLOB',
3998 'big-id': 'BIGINT not null unique with default next value for %s'% INGRES_SEQNAME,
3999 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4000 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4001 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
4002 }
4003
4005 drivers = ('sapdb',)
4006
4007 support_distributed_transaction = False
4008 types = {
4009 'boolean': 'CHAR(1)',
4010 'string': 'VARCHAR(%(length)s)',
4011 'text': 'LONG',
4012 'json': 'LONG',
4013 'password': 'VARCHAR(%(length)s)',
4014 'blob': 'LONG',
4015 'upload': 'VARCHAR(%(length)s)',
4016 'integer': 'INT',
4017 'bigint': 'BIGINT',
4018 'float': 'FLOAT',
4019 'double': 'DOUBLE PRECISION',
4020 'decimal': 'FIXED(%(precision)s,%(scale)s)',
4021 'date': 'DATE',
4022 'time': 'TIME',
4023 'datetime': 'TIMESTAMP',
4024 'id': 'INT PRIMARY KEY',
4025 'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4026 'list:integer': 'LONG',
4027 'list:string': 'LONG',
4028 'list:reference': 'LONG',
4029 'big-id': 'BIGINT PRIMARY KEY',
4030 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4031 }
4032
4034 return '%s_id_Seq' % table
4035
4036 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
4037 if limitby:
4038 (lmin, lmax) = limitby
4039 if len(sql_w) > 1:
4040 sql_w_row = sql_w + ' AND w_row > %i' % lmin
4041 else:
4042 sql_w_row = 'WHERE w_row > %i' % lmin
4043 return '%s %s FROM (SELECT w_tmp.*, ROWNO w_row FROM (SELECT %s FROM %s%s%s) w_tmp WHERE ROWNO=%i) %s %s %s;' % (sql_s, sql_f, sql_f, sql_t, sql_w, sql_o, lmax, sql_t, sql_w_row, sql_o)
4044 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
4045
4047
4048 self.execute('CREATE SEQUENCE %s;' % table._sequence_name)
4049 self.execute("ALTER TABLE %s ALTER COLUMN %s SET DEFAULT NEXTVAL('%s');" \
4050 % (table._tablename, table._id.name, table._sequence_name))
4051 self.execute(query)
4052
4053 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?sslmode=(?P<sslmode>.+))?$')
4054
4055
4056 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
4057 credential_decoder=IDENTITY, driver_args={},
4058 adapter_args={}, do_connect=True, after_connection=None):
4059 self.db = db
4060 self.dbengine = "sapdb"
4061 self.uri = uri
4062 if do_connect: self.find_driver(adapter_args,uri)
4063 self.pool_size = pool_size
4064 self.folder = folder
4065 self.db_codec = db_codec
4066 self._after_connection = after_connection
4067 self.find_or_make_work_folder()
4068 ruri = uri.split('://',1)[1]
4069 m = self.REGEX_URI.match(ruri)
4070 if not m:
4071 raise SyntaxError("Invalid URI string in DAL")
4072 user = credential_decoder(m.group('user'))
4073 if not user:
4074 raise SyntaxError('User required')
4075 password = credential_decoder(m.group('password'))
4076 if not password:
4077 password = ''
4078 host = m.group('host')
4079 if not host:
4080 raise SyntaxError('Host name required')
4081 db = m.group('db')
4082 if not db:
4083 raise SyntaxError('Database name required')
4084 def connector(user=user, password=password, database=db,
4085 host=host, driver_args=driver_args):
4086 return self.driver.Connection(user, password, database,
4087 host, **driver_args)
4088 self.connector = connector
4089 if do_connect: self.reconnect()
4090
4092 self.execute("select %s.NEXTVAL from dual" % table._sequence_name)
4093 return int(self.cursor.fetchone()[0])
4094
4096 drivers = ('cubriddb',)
4097
4098 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^?]+)(\?set_encoding=(?P<charset>\w+))?$')
4099
4100 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec='UTF-8',
4101 credential_decoder=IDENTITY, driver_args={},
4102 adapter_args={}, do_connect=True, after_connection=None):
4103 self.db = db
4104 self.dbengine = "cubrid"
4105 self.uri = uri
4106 if do_connect: self.find_driver(adapter_args,uri)
4107 self.pool_size = pool_size
4108 self.folder = folder
4109 self.db_codec = db_codec
4110 self._after_connection = after_connection
4111 self.find_or_make_work_folder()
4112 ruri = uri.split('://',1)[1]
4113 m = self.REGEX_URI.match(ruri)
4114 if not m:
4115 raise SyntaxError(
4116 "Invalid URI string in DAL: %s" % self.uri)
4117 user = credential_decoder(m.group('user'))
4118 if not user:
4119 raise SyntaxError('User required')
4120 password = credential_decoder(m.group('password'))
4121 if not password:
4122 password = ''
4123 host = m.group('host')
4124 if not host:
4125 raise SyntaxError('Host name required')
4126 db = m.group('db')
4127 if not db:
4128 raise SyntaxError('Database name required')
4129 port = int(m.group('port') or '30000')
4130 charset = m.group('charset') or 'utf8'
4131 user = credential_decoder(user)
4132 passwd = credential_decoder(password)
4133 def connector(host=host,port=port,db=db,
4134 user=user,passwd=password,driver_args=driver_args):
4135 return self.driver.connect(host,port,db,user,passwd,**driver_args)
4136 self.connector = connector
4137 if do_connect: self.reconnect()
4138
4140 self.execute('SET FOREIGN_KEY_CHECKS=1;')
4141 self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
4142
4147
4148 web2py_filesystem = False
4149
4151 return self.db._adapter.escape(obj)
4152
4154 if not db._adapter.dbengine in ('mysql', 'postgres'):
4155 raise RuntimeError("only MySQL/Postgres can store metadata .table files in database for now")
4156 self.db = db
4157 self.filename = filename
4158 self.mode = mode
4159 if not self.web2py_filesystem:
4160 if db._adapter.dbengine == 'mysql':
4161 sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content LONGTEXT, PRIMARY KEY(path) ) ENGINE=InnoDB;"
4162 elif db._adapter.dbengine == 'postgres':
4163 sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content TEXT, PRIMARY KEY(path));"
4164 self.db.executesql(sql)
4165 DatabaseStoredFile.web2py_filesystem = True
4166 self.p=0
4167 self.data = ''
4168 if mode in ('r','rw','a'):
4169 query = "SELECT content FROM web2py_filesystem WHERE path='%s'" \
4170 % filename
4171 rows = self.db.executesql(query)
4172 if rows:
4173 self.data = rows[0][0]
4174 elif exists(filename):
4175 datafile = open(filename, 'r')
4176 try:
4177 self.data = datafile.read()
4178 finally:
4179 datafile.close()
4180 elif mode in ('r','rw'):
4181 raise RuntimeError("File %s does not exist" % filename)
4182
4183 - def read(self, bytes):
4184 data = self.data[self.p:self.p+bytes]
4185 self.p += len(data)
4186 return data
4187
4189 i = self.data.find('\n',self.p)+1
4190 if i>0:
4191 data, self.p = self.data[self.p:i], i
4192 else:
4193 data, self.p = self.data[self.p:], len(self.data)
4194 return data
4195
4198
4200 if self.db is not None:
4201 self.db.executesql(
4202 "DELETE FROM web2py_filesystem WHERE path='%s'" % self.filename)
4203 query = "INSERT INTO web2py_filesystem(path,content) VALUES ('%s','%s')"\
4204 % (self.filename, self.data.replace("'","''"))
4205 self.db.executesql(query)
4206 self.db.commit()
4207 self.db = None
4208
4211
4212 @staticmethod
4214 if exists(filename):
4215 return True
4216 query = "SELECT path FROM web2py_filesystem WHERE path='%s'" % filename
4217 if db.executesql(query):
4218 return True
4219 return False
4220
4223
4226
4227 - def file_open(self, filename, mode='rb', lock=True):
4229
4232
4234 query = "DELETE FROM web2py_filesystem WHERE path='%s'" % filename
4235 self.db.executesql(query)
4236 self.db.commit()
4237
4239 uploads_in_blob = True
4240
4241 REGEX_URI = re.compile('^(?P<instance>.*)/(?P<db>.*)$')
4242
4243 - def __init__(self, db, uri='google:sql://realm:domain/database',
4244 pool_size=0, folder=None, db_codec='UTF-8',
4245 credential_decoder=IDENTITY, driver_args={},
4246 adapter_args={}, do_connect=True, after_connection=None):
4247
4248 self.db = db
4249 self.dbengine = "mysql"
4250 self.uri = uri
4251 self.pool_size = pool_size
4252 self.db_codec = db_codec
4253 self._after_connection = after_connection
4254 self.folder = folder or pjoin('$HOME',THREAD_LOCAL.folder.split(
4255 os.sep+'applications'+os.sep,1)[1])
4256 ruri = uri.split("://")[1]
4257 m = self.REGEX_URI.match(ruri)
4258 if not m:
4259 raise SyntaxError("Invalid URI string in SQLDB: %s" % self.uri)
4260 instance = credential_decoder(m.group('instance'))
4261 self.dbstring = db = credential_decoder(m.group('db'))
4262 driver_args['instance'] = instance
4263 if not 'charset' in driver_args:
4264 driver_args['charset'] = 'utf8'
4265 self.createdb = createdb = adapter_args.get('createdb',True)
4266 if not createdb:
4267 driver_args['database'] = db
4268 def connector(driver_args=driver_args):
4269 return rdbms.connect(**driver_args)
4270 self.connector = connector
4271 if do_connect: self.reconnect()
4272
4274 if self.createdb:
4275
4276 self.execute('CREATE DATABASE IF NOT EXISTS %s' % self.dbstring)
4277 self.execute('USE %s' % self.dbstring)
4278 self.execute("SET FOREIGN_KEY_CHECKS=1;")
4279 self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
4280
4281 - def execute(self, command, *a, **b):
4283
4285 can_select_for_update = False
4286
4287 @staticmethod
4289 if isinstance(obj, str):
4290 return obj.decode('utf8')
4291 elif not isinstance(obj, unicode):
4292 return unicode(obj)
4293 return obj
4294
4296 return table._id > 0
4297
4299 field_is_type = fieldtype.startswith
4300 if isinstance(obj, CALLABLETYPES):
4301 obj = obj()
4302 if isinstance(fieldtype, SQLCustomType):
4303 return fieldtype.encoder(obj)
4304 if isinstance(obj, (Expression, Field)):
4305 raise SyntaxError("non supported on GAE")
4306 if self.dbengine == 'google:datastore':
4307 if isinstance(fieldtype, gae.Property):
4308 return obj
4309 is_string = isinstance(fieldtype,str)
4310 is_list = is_string and field_is_type('list:')
4311 if is_list:
4312 if not obj:
4313 obj = []
4314 if not isinstance(obj, (list, tuple)):
4315 obj = [obj]
4316 if obj == '' and not \
4317 (is_string and fieldtype[:2] in ['st','te', 'pa','up']):
4318 return None
4319 if not obj is None:
4320 if isinstance(obj, list) and not is_list:
4321 obj = [self.represent(o, fieldtype) for o in obj]
4322 elif fieldtype in ('integer','bigint','id'):
4323 obj = long(obj)
4324 elif fieldtype == 'double':
4325 obj = float(obj)
4326 elif is_string and field_is_type('reference'):
4327 if isinstance(obj, (Row, Reference)):
4328 obj = obj['id']
4329 obj = long(obj)
4330 elif fieldtype == 'boolean':
4331 if obj and not str(obj)[0].upper() in '0F':
4332 obj = True
4333 else:
4334 obj = False
4335 elif fieldtype == 'date':
4336 if not isinstance(obj, datetime.date):
4337 (y, m, d) = map(int,str(obj).strip().split('-'))
4338 obj = datetime.date(y, m, d)
4339 elif isinstance(obj,datetime.datetime):
4340 (y, m, d) = (obj.year, obj.month, obj.day)
4341 obj = datetime.date(y, m, d)
4342 elif fieldtype == 'time':
4343 if not isinstance(obj, datetime.time):
4344 time_items = map(int,str(obj).strip().split(':')[:3])
4345 if len(time_items) == 3:
4346 (h, mi, s) = time_items
4347 else:
4348 (h, mi, s) = time_items + [0]
4349 obj = datetime.time(h, mi, s)
4350 elif fieldtype == 'datetime':
4351 if not isinstance(obj, datetime.datetime):
4352 (y, m, d) = map(int,str(obj)[:10].strip().split('-'))
4353 time_items = map(int,str(obj)[11:].strip().split(':')[:3])
4354 while len(time_items)<3:
4355 time_items.append(0)
4356 (h, mi, s) = time_items
4357 obj = datetime.datetime(y, m, d, h, mi, s)
4358 elif fieldtype == 'blob':
4359 pass
4360 elif fieldtype == 'json':
4361 obj = self.to_unicode(obj)
4362 if have_serializers:
4363 obj = serializers.loads_json(obj)
4364 elif simplejson:
4365 obj = simplejson.loads(obj)
4366 else:
4367 raise RuntimeError("missing simplejson")
4368 elif is_string and field_is_type('list:string'):
4369 return map(self.to_unicode,obj)
4370 elif is_list:
4371 return map(int,obj)
4372 else:
4373 obj = self.to_unicode(obj)
4374 return obj
4375
4377 return 'insert %s in %s' % (fields, table)
4378
4379 - def _count(self,query,distinct=None):
4380 return 'count %s' % repr(query)
4381
4382 - def _select(self,query,fields,attributes):
4383 return 'select %s where %s' % (repr(fields), repr(query))
4384
4385 - def _delete(self,tablename, query):
4386 return 'delete %s where %s' % (repr(tablename),repr(query))
4387
4388 - def _update(self,tablename,query,fields):
4389 return 'update %s (%s) where %s' % (repr(tablename),
4390 repr(fields),repr(query))
4391
4393 """
4394 remember: no transactions on many NoSQL
4395 """
4396 pass
4397
4399 """
4400 remember: no transactions on many NoSQL
4401 """
4402 pass
4403
4405 """
4406 remember: no transactions on many NoSQL
4407 """
4408 pass
4409
4410
4411
4412 - def OR(self,first,second): raise SyntaxError("Not supported")
4413 - def AND(self,first,second): raise SyntaxError("Not supported")
4414 - def AS(self,first,second): raise SyntaxError("Not supported")
4415 - def ON(self,first,second): raise SyntaxError("Not supported")
4416 - def STARTSWITH(self,first,second=None): raise SyntaxError("Not supported")
4417 - def ENDSWITH(self,first,second=None): raise SyntaxError("Not supported")
4418 - def ADD(self,first,second): raise SyntaxError("Not supported")
4419 - def SUB(self,first,second): raise SyntaxError("Not supported")
4420 - def MUL(self,first,second): raise SyntaxError("Not supported")
4421 - def DIV(self,first,second): raise SyntaxError("Not supported")
4422 - def LOWER(self,first): raise SyntaxError("Not supported")
4423 - def UPPER(self,first): raise SyntaxError("Not supported")
4425 - def AGGREGATE(self,first,what): raise SyntaxError("Not supported")
4426 - def LEFT_JOIN(self): raise SyntaxError("Not supported")
4427 - def RANDOM(self): raise SyntaxError("Not supported")
4428 - def SUBSTRING(self,field,parameters): raise SyntaxError("Not supported")
4429 - def PRIMARY_KEY(self,key): raise SyntaxError("Not supported")
4430 - def ILIKE(self,first,second): raise SyntaxError("Not supported")
4431 - def drop(self,table,mode): raise SyntaxError("Not supported")
4432 - def alias(self,table,alias): raise SyntaxError("Not supported")
4433 - def migrate_table(self,*a,**b): raise SyntaxError("Not supported")
4435 - def prepare(self,key): raise SyntaxError("Not supported")
4438 - def concat_add(self,table): raise SyntaxError("Not supported")
4439 - def constraint_name(self, table, fieldname): raise SyntaxError("Not supported")
4441 - def log_execute(self,*a,**b): raise SyntaxError("Not supported")
4442 - def execute(self,*a,**b): raise SyntaxError("Not supported")
4444 - def lastrowid(self,table): raise SyntaxError("Not supported")
4446 - def rowslice(self,rows,minimum=0,maximum=None): raise SyntaxError("Not supported")
4447
4448
4449 -class GAEF(object):
4450 - def __init__(self,name,op,value,apply):
4451 self.name=name=='id' and '__key__' or name
4452 self.op=op
4453 self.value=value
4454 self.apply=apply
4456 return '(%s %s %s:%s)' % (self.name, self.op, repr(self.value), type(self.value))
4457
4459 uploads_in_blob = True
4460 types = {}
4461
4463 - def file_open(self, filename, mode='rb', lock=True): pass
4465
4466 REGEX_NAMESPACE = re.compile('.*://(?P<namespace>.+)')
4467
4468 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
4469 credential_decoder=IDENTITY, driver_args={},
4470 adapter_args={}, do_connect=True, after_connection=None):
4471 self.types.update({
4472 'boolean': gae.BooleanProperty,
4473 'string': (lambda **kwargs: gae.StringProperty(multiline=True, **kwargs)),
4474 'text': gae.TextProperty,
4475 'json': gae.TextProperty,
4476 'password': gae.StringProperty,
4477 'blob': gae.BlobProperty,
4478 'upload': gae.StringProperty,
4479 'integer': gae.IntegerProperty,
4480 'bigint': gae.IntegerProperty,
4481 'float': gae.FloatProperty,
4482 'double': gae.FloatProperty,
4483 'decimal': GAEDecimalProperty,
4484 'date': gae.DateProperty,
4485 'time': gae.TimeProperty,
4486 'datetime': gae.DateTimeProperty,
4487 'id': None,
4488 'reference': gae.IntegerProperty,
4489 'list:string': (lambda **kwargs: gae.StringListProperty(default=None, **kwargs)),
4490 'list:integer': (lambda **kwargs: gae.ListProperty(int,default=None, **kwargs)),
4491 'list:reference': (lambda **kwargs: gae.ListProperty(int,default=None, **kwargs)),
4492 })
4493 self.db = db
4494 self.uri = uri
4495 self.dbengine = 'google:datastore'
4496 self.folder = folder
4497 db['_lastsql'] = ''
4498 self.db_codec = 'UTF-8'
4499 self._after_connection = after_connection
4500 self.pool_size = 0
4501 match = self.REGEX_NAMESPACE.match(uri)
4502 if match:
4503 namespace_manager.set_namespace(match.group('namespace'))
4504
4505 - def parse_id(self, value, field_type):
4507
4508 - def create_table(self,table,migrate=True,fake_migrate=False, polymodel=None):
4509 myfields = {}
4510 for field in table:
4511 if isinstance(polymodel,Table) and field.name in polymodel.fields():
4512 continue
4513 attr = {}
4514 if isinstance(field.custom_qualifier, dict):
4515
4516 attr = field.custom_qualifier
4517 field_type = field.type
4518 if isinstance(field_type, SQLCustomType):
4519 ftype = self.types[field_type.native or field_type.type](**attr)
4520 elif isinstance(field_type, gae.Property):
4521 ftype = field_type
4522 elif field_type.startswith('id'):
4523 continue
4524 elif field_type.startswith('decimal'):
4525 precision, scale = field_type[7:].strip('()').split(',')
4526 precision = int(precision)
4527 scale = int(scale)
4528 ftype = GAEDecimalProperty(precision, scale, **attr)
4529 elif field_type.startswith('reference'):
4530 if field.notnull:
4531 attr = dict(required=True)
4532 referenced = field_type[10:].strip()
4533 ftype = self.types[field_type[:9]](referenced, **attr)
4534 elif field_type.startswith('list:reference'):
4535 if field.notnull:
4536 attr['required'] = True
4537 referenced = field_type[15:].strip()
4538 ftype = self.types[field_type[:14]](**attr)
4539 elif field_type.startswith('list:'):
4540 ftype = self.types[field_type](**attr)
4541 elif not field_type in self.types\
4542 or not self.types[field_type]:
4543 raise SyntaxError('Field: unknown field type: %s' % field_type)
4544 else:
4545 ftype = self.types[field_type](**attr)
4546 myfields[field.name] = ftype
4547 if not polymodel:
4548 table._tableobj = classobj(table._tablename, (gae.Model, ), myfields)
4549 elif polymodel==True:
4550 table._tableobj = classobj(table._tablename, (PolyModel, ), myfields)
4551 elif isinstance(polymodel,Table):
4552 table._tableobj = classobj(table._tablename, (polymodel._tableobj, ), myfields)
4553 else:
4554 raise SyntaxError("polymodel must be None, True, a table or a tablename")
4555 return None
4556
4557 - def expand(self,expression,field_type=None):
4558 if isinstance(expression,Field):
4559 if expression.type in ('text', 'blob', 'json'):
4560 raise SyntaxError('AppEngine does not index by: %s' % expression.type)
4561 return expression.name
4562 elif isinstance(expression, (Expression, Query)):
4563 if not expression.second is None:
4564 return expression.op(expression.first, expression.second)
4565 elif not expression.first is None:
4566 return expression.op(expression.first)
4567 else:
4568 return expression.op()
4569 elif field_type:
4570 return self.represent(expression,field_type)
4571 elif isinstance(expression,(list,tuple)):
4572 return ','.join([self.represent(item,field_type) for item in expression])
4573 else:
4574 return str(expression)
4575
4576
4577 - def AND(self,first,second):
4583
4584 - def EQ(self,first,second=None):
4585 if isinstance(second, Key):
4586 return [GAEF(first.name,'=',second,lambda a,b:a==b)]
4587 return [GAEF(first.name,'=',self.represent(second,first.type),lambda a,b:a==b)]
4588
4589 - def NE(self,first,second=None):
4590 if first.type != 'id':
4591 return [GAEF(first.name,'!=',self.represent(second,first.type),lambda a,b:a!=b)]
4592 else:
4593 if not second is None:
4594 second = Key.from_path(first._tablename, long(second))
4595 return [GAEF(first.name,'!=',second,lambda a,b:a!=b)]
4596
4597 - def LT(self,first,second=None):
4598 if first.type != 'id':
4599 return [GAEF(first.name,'<',self.represent(second,first.type),lambda a,b:a<b)]
4600 else:
4601 second = Key.from_path(first._tablename, long(second))
4602 return [GAEF(first.name,'<',second,lambda a,b:a<b)]
4603
4604 - def LE(self,first,second=None):
4605 if first.type != 'id':
4606 return [GAEF(first.name,'<=',self.represent(second,first.type),lambda a,b:a<=b)]
4607 else:
4608 second = Key.from_path(first._tablename, long(second))
4609 return [GAEF(first.name,'<=',second,lambda a,b:a<=b)]
4610
4611 - def GT(self,first,second=None):
4612 if first.type != 'id' or second==0 or second == '0':
4613 return [GAEF(first.name,'>',self.represent(second,first.type),lambda a,b:a>b)]
4614 else:
4615 second = Key.from_path(first._tablename, long(second))
4616 return [GAEF(first.name,'>',second,lambda a,b:a>b)]
4617
4618 - def GE(self,first,second=None):
4619 if first.type != 'id':
4620 return [GAEF(first.name,'>=',self.represent(second,first.type),lambda a,b:a>=b)]
4621 else:
4622 second = Key.from_path(first._tablename, long(second))
4623 return [GAEF(first.name,'>=',second,lambda a,b:a>=b)]
4624
4627
4628 - def COMMA(self,first,second):
4630
4631 - def BELONGS(self,first,second=None):
4632 if not isinstance(second,(list, tuple)):
4633 raise SyntaxError("Not supported")
4634 if first.type != 'id':
4635 return [GAEF(first.name,'in',self.represent(second,first.type),lambda a,b:a in b)]
4636 else:
4637 second = [Key.from_path(first._tablename, int(i)) for i in second]
4638 return [GAEF(first.name,'in',second,lambda a,b:a in b)]
4639
4640 - def CONTAINS(self,first,second,case_sensitive=False):
4645
4646 - def NOT(self,first):
4647 nops = { self.EQ: self.NE,
4648 self.NE: self.EQ,
4649 self.LT: self.GE,
4650 self.GT: self.LE,
4651 self.LE: self.GT,
4652 self.GE: self.LT}
4653 if not isinstance(first,Query):
4654 raise SyntaxError("Not suported")
4655 nop = nops.get(first.op,None)
4656 if not nop:
4657 raise SyntaxError("Not suported %s" % first.op.__name__)
4658 first.op = nop
4659 return self.expand(first)
4660
4663
4664 - def select_raw(self,query,fields=None,attributes=None):
4665 db = self.db
4666 fields = fields or []
4667 attributes = attributes or {}
4668 args_get = attributes.get
4669 new_fields = []
4670 for item in fields:
4671 if isinstance(item,SQLALL):
4672 new_fields += item._table
4673 else:
4674 new_fields.append(item)
4675 fields = new_fields
4676 if query:
4677 tablename = self.get_table(query)
4678 elif fields:
4679 tablename = fields[0].tablename
4680 query = db._adapter.id_query(fields[0].table)
4681 else:
4682 raise SyntaxError("Unable to determine a tablename")
4683
4684 if query:
4685 if use_common_filters(query):
4686 query = self.common_filter(query,[tablename])
4687
4688
4689 tableobj = db[tablename]._tableobj
4690 filters = self.expand(query)
4691
4692 projection = None
4693 if len(db[tablename].fields) == len(fields):
4694
4695 projection = None
4696 elif args_get('projection') == True:
4697 projection = []
4698 for f in fields:
4699 if f.type in ['text', 'blob', 'json']:
4700 raise SyntaxError(
4701 "text and blob field types not allowed in projection queries")
4702 else:
4703 projection.append(f.name)
4704
4705
4706
4707 query_projection = [
4708 p for p in projection if \
4709 p != db[tablename]._id.name] if projection \
4710 else None
4711
4712 cursor = None
4713 if isinstance(args_get('reusecursor'), str):
4714 cursor = args_get('reusecursor')
4715 items = gae.Query(tableobj, projection=query_projection,
4716 cursor=cursor)
4717
4718 for filter in filters:
4719 if args_get('projection') == True and \
4720 filter.name in query_projection and \
4721 filter.op in ['=', '<=', '>=']:
4722 raise SyntaxError(
4723 "projection fields cannot have equality filters")
4724 if filter.name=='__key__' and filter.op=='>' and filter.value==0:
4725 continue
4726 elif filter.name=='__key__' and filter.op=='=':
4727 if filter.value==0:
4728 items = []
4729 elif isinstance(filter.value, Key):
4730
4731
4732
4733 item = tableobj.get(filter.value)
4734 items = (item and [item]) or []
4735 else:
4736
4737
4738
4739 item = tableobj.get_by_id(filter.value)
4740 items = (item and [item]) or []
4741 elif isinstance(items,list):
4742 items = [i for i in items if filter.apply(
4743 getattr(item,filter.name),filter.value)]
4744 else:
4745 if filter.name=='__key__' and filter.op != 'in':
4746 items.order('__key__')
4747 items = items.filter('%s %s' % (filter.name,filter.op),
4748 filter.value)
4749 if not isinstance(items,list):
4750 if args_get('left', None):
4751 raise SyntaxError('Set: no left join in appengine')
4752 if args_get('groupby', None):
4753 raise SyntaxError('Set: no groupby in appengine')
4754 orderby = args_get('orderby', False)
4755 if orderby:
4756
4757 if isinstance(orderby, (list, tuple)):
4758 orderby = xorify(orderby)
4759 if isinstance(orderby,Expression):
4760 orderby = self.expand(orderby)
4761 orders = orderby.split(', ')
4762 for order in orders:
4763 order={'-id':'-__key__','id':'__key__'}.get(order,order)
4764 items = items.order(order)
4765 if args_get('limitby', None):
4766 (lmin, lmax) = attributes['limitby']
4767 (limit, offset) = (lmax - lmin, lmin)
4768 rows = items.fetch(limit,offset=offset)
4769
4770
4771 if args_get('reusecursor'):
4772 db['_lastcursor'] = items.cursor()
4773 items = rows
4774 return (items, tablename, projection or db[tablename].fields)
4775
4776 - def select(self,query,fields,attributes):
4777 """
4778 This is the GAE version of select. some notes to consider:
4779 - db['_lastsql'] is not set because there is not SQL statement string
4780 for a GAE query
4781 - 'nativeRef' is a magical fieldname used for self references on GAE
4782 - optional attribute 'projection' when set to True will trigger
4783 use of the GAE projection queries. note that there are rules for
4784 what is accepted imposed by GAE: each field must be indexed,
4785 projection queries cannot contain blob or text fields, and you
4786 cannot use == and also select that same field. see https://developers.google.com/appengine/docs/python/datastore/queries#Query_Projection
4787 - optional attribute 'reusecursor' allows use of cursor with queries
4788 that have the limitby attribute. Set the attribute to True for the
4789 first query, set it to the value of db['_lastcursor'] to continue
4790 a previous query. The user must save the cursor value between
4791 requests, and the filters must be identical. It is up to the user
4792 to follow google's limitations: https://developers.google.com/appengine/docs/python/datastore/queries#Query_Cursors
4793 """
4794
4795 (items, tablename, fields) = self.select_raw(query,fields,attributes)
4796
4797 rows = [[(t==self.db[tablename]._id.name and item) or \
4798 (t=='nativeRef' and item) or getattr(item, t) \
4799 for t in fields] for item in items]
4800 colnames = ['%s.%s' % (tablename, t) for t in fields]
4801 processor = attributes.get('processor',self.parse)
4802 return processor(rows,fields,colnames,False)
4803
4804 - def count(self,query,distinct=None,limit=None):
4805 if distinct:
4806 raise RuntimeError("COUNT DISTINCT not supported")
4807 (items, tablename, fields) = self.select_raw(query)
4808
4809 try:
4810 return len(items)
4811 except TypeError:
4812 return items.count(limit=limit)
4813
4814 - def delete(self,tablename, query):
4815 """
4816 This function was changed on 2010-05-04 because according to
4817 http://code.google.com/p/googleappengine/issues/detail?id=3119
4818 GAE no longer supports deleting more than 1000 records.
4819 """
4820
4821 (items, tablename, fields) = self.select_raw(query)
4822
4823 if not isinstance(items,list):
4824
4825
4826 leftitems = items.fetch(1000, keys_only=True)
4827 counter = 0
4828 while len(leftitems):
4829 counter += len(leftitems)
4830 gae.delete(leftitems)
4831 leftitems = items.fetch(1000, keys_only=True)
4832 else:
4833 counter = len(items)
4834 gae.delete(items)
4835 return counter
4836
4837 - def update(self,tablename,query,update_fields):
4838
4839 (items, tablename, fields) = self.select_raw(query)
4840 counter = 0
4841 for item in items:
4842 for field, value in update_fields:
4843 setattr(item, field.name, self.represent(value,field.type))
4844 item.put()
4845 counter += 1
4846 LOGGER.info(str(counter))
4847 return counter
4848
4849 - def insert(self,table,fields):
4850 dfields=dict((f.name,self.represent(v,f.type)) for f,v in fields)
4851
4852 tmp = table._tableobj(**dfields)
4853 tmp.put()
4854 rid = Reference(tmp.key().id())
4855 (rid._table, rid._record, rid._gaekey) = (table, None, tmp.key())
4856 return rid
4857
4859 parsed_items = []
4860 for item in items:
4861 dfields=dict((f.name,self.represent(v,f.type)) for f,v in item)
4862 parsed_items.append(table._tableobj(**dfields))
4863 gae.put(parsed_items)
4864 return True
4865
4867 return uuid.UUID(uuidv).int
4868
4870 return str(uuid.UUID(int=n))
4871
4873 drivers = ('couchdb',)
4874
4875 uploads_in_blob = True
4876 types = {
4877 'boolean': bool,
4878 'string': str,
4879 'text': str,
4880 'json': str,
4881 'password': str,
4882 'blob': str,
4883 'upload': str,
4884 'integer': long,
4885 'bigint': long,
4886 'float': float,
4887 'double': float,
4888 'date': datetime.date,
4889 'time': datetime.time,
4890 'datetime': datetime.datetime,
4891 'id': long,
4892 'reference': long,
4893 'list:string': list,
4894 'list:integer': list,
4895 'list:reference': list,
4896 }
4897
4899 - def file_open(self, filename, mode='rb', lock=True): pass
4901
4902 - def expand(self,expression,field_type=None):
4903 if isinstance(expression,Field):
4904 if expression.type=='id':
4905 return "%s._id" % expression.tablename
4906 return BaseAdapter.expand(self,expression,field_type)
4907
4908 - def AND(self,first,second):
4910
4911 - def OR(self,first,second):
4913
4914 - def EQ(self,first,second):
4918
4919 - def NE(self,first,second):
4923
4924 - def COMMA(self,first,second):
4926
4928 value = NoSQLAdapter.represent(self, obj, fieldtype)
4929 if fieldtype=='id':
4930 return repr(str(int(value)))
4931 elif fieldtype in ('date','time','datetime','boolean'):
4932 return serializers.json(value)
4933 return repr(not isinstance(value,unicode) and value \
4934 or value and value.encode('utf8'))
4935
4936 - def __init__(self,db,uri='couchdb://127.0.0.1:5984',
4937 pool_size=0,folder=None,db_codec ='UTF-8',
4938 credential_decoder=IDENTITY, driver_args={},
4939 adapter_args={}, do_connect=True, after_connection=None):
4940 self.db = db
4941 self.uri = uri
4942 if do_connect: self.find_driver(adapter_args)
4943 self.dbengine = 'couchdb'
4944 self.folder = folder
4945 db['_lastsql'] = ''
4946 self.db_codec = 'UTF-8'
4947 self._after_connection = after_connection
4948 self.pool_size = pool_size
4949
4950 url='http://'+uri[10:]
4951 def connector(url=url,driver_args=driver_args):
4952 return self.driver.Server(url,**driver_args)
4953 self.reconnect(connector,cursor=False)
4954
4955 - def create_table(self, table, migrate=True, fake_migrate=False, polymodel=None):
4956 if migrate:
4957 try:
4958 self.connection.create(table._tablename)
4959 except:
4960 pass
4961
4962 - def insert(self,table,fields):
4969
4970 - def _select(self,query,fields,attributes):
4971 if not isinstance(query,Query):
4972 raise SyntaxError("Not Supported")
4973 for key in set(attributes.keys())-SELECT_ARGS:
4974 raise SyntaxError('invalid select attribute: %s' % key)
4975 new_fields=[]
4976 for item in fields:
4977 if isinstance(item,SQLALL):
4978 new_fields += item._table
4979 else:
4980 new_fields.append(item)
4981 def uid(fd):
4982 return fd=='id' and '_id' or fd
4983 def get(row,fd):
4984 return fd=='id' and int(row['_id']) or row.get(fd,None)
4985 fields = new_fields
4986 tablename = self.get_table(query)
4987 fieldnames = [f.name for f in (fields or self.db[tablename])]
4988 colnames = ['%s.%s' % (tablename,k) for k in fieldnames]
4989 fields = ','.join(['%s.%s' % (tablename,uid(f)) for f in fieldnames])
4990 fn="(function(%(t)s){if(%(query)s)emit(%(order)s,[%(fields)s]);})" %\
4991 dict(t=tablename,
4992 query=self.expand(query),
4993 order='%s._id' % tablename,
4994 fields=fields)
4995 return fn, colnames
4996
4997 - def select(self,query,fields,attributes):
4998 if not isinstance(query,Query):
4999 raise SyntaxError("Not Supported")
5000 fn, colnames = self._select(query,fields,attributes)
5001 tablename = colnames[0].split('.')[0]
5002 ctable = self.connection[tablename]
5003 rows = [cols['value'] for cols in ctable.query(fn)]
5004 processor = attributes.get('processor',self.parse)
5005 return processor(rows,fields,colnames,False)
5006
5007 - def delete(self,tablename,query):
5008 if not isinstance(query,Query):
5009 raise SyntaxError("Not Supported")
5010 if query.first.type=='id' and query.op==self.EQ:
5011 id = query.second
5012 tablename = query.first.tablename
5013 assert(tablename == query.first.tablename)
5014 ctable = self.connection[tablename]
5015 try:
5016 del ctable[str(id)]
5017 return 1
5018 except couchdb.http.ResourceNotFound:
5019 return 0
5020 else:
5021 tablename = self.get_table(query)
5022 rows = self.select(query,[self.db[tablename]._id],{})
5023 ctable = self.connection[tablename]
5024 for row in rows:
5025 del ctable[str(row.id)]
5026 return len(rows)
5027
5028 - def update(self,tablename,query,fields):
5029 if not isinstance(query,Query):
5030 raise SyntaxError("Not Supported")
5031 if query.first.type=='id' and query.op==self.EQ:
5032 id = query.second
5033 tablename = query.first.tablename
5034 ctable = self.connection[tablename]
5035 try:
5036 doc = ctable[str(id)]
5037 for key,value in fields:
5038 doc[key.name] = self.represent(value,self.db[tablename][key.name].type)
5039 ctable.save(doc)
5040 return 1
5041 except couchdb.http.ResourceNotFound:
5042 return 0
5043 else:
5044 tablename = self.get_table(query)
5045 rows = self.select(query,[self.db[tablename]._id],{})
5046 ctable = self.connection[tablename]
5047 table = self.db[tablename]
5048 for row in rows:
5049 doc = ctable[str(row.id)]
5050 for key,value in fields:
5051 doc[key.name] = self.represent(value,table[key.name].type)
5052 ctable.save(doc)
5053 return len(rows)
5054
5055 - def count(self,query,distinct=None):
5056 if distinct:
5057 raise RuntimeError("COUNT DISTINCT not supported")
5058 if not isinstance(query,Query):
5059 raise SyntaxError("Not Supported")
5060 tablename = self.get_table(query)
5061 rows = self.select(query,[self.db[tablename]._id],{})
5062 return len(rows)
5063
5065 """
5066 validates that the given text is clean: only contains [0-9a-zA-Z_]
5067 """
5068 if not REGEX_ALPHANUMERIC.match(text):
5069 raise SyntaxError('invalid table or field name: %s' % text)
5070 return text
5071
5073 native_json = True
5074 drivers = ('pymongo',)
5075
5076 uploads_in_blob = True
5077
5078 types = {
5079 'boolean': bool,
5080 'string': str,
5081 'text': str,
5082 'json': str,
5083 'password': str,
5084 'blob': str,
5085 'upload': str,
5086 'integer': long,
5087 'bigint': long,
5088 'float': float,
5089 'double': float,
5090 'date': datetime.date,
5091 'time': datetime.time,
5092 'datetime': datetime.datetime,
5093 'id': long,
5094 'reference': long,
5095 'list:string': list,
5096 'list:integer': list,
5097 'list:reference': list,
5098 }
5099
5100 error_messages = {"javascript_needed": "This must yet be replaced" +
5101 " with javascript in order to work."}
5102
5103 - def __init__(self,db,uri='mongodb://127.0.0.1:5984/db',
5104 pool_size=0, folder=None, db_codec ='UTF-8',
5105 credential_decoder=IDENTITY, driver_args={},
5106 adapter_args={}, do_connect=True, after_connection=None):
5107
5108 self.db = db
5109 self.uri = uri
5110 if do_connect: self.find_driver(adapter_args)
5111 import random
5112 from bson.objectid import ObjectId
5113 from bson.son import SON
5114 import pymongo.uri_parser
5115
5116 m = pymongo.uri_parser.parse_uri(uri)
5117
5118 self.SON = SON
5119 self.ObjectId = ObjectId
5120 self.random = random
5121
5122 self.dbengine = 'mongodb'
5123 self.folder = folder
5124 db['_lastsql'] = ''
5125 self.db_codec = 'UTF-8'
5126 self._after_connection = after_connection
5127 self.pool_size = pool_size
5128
5129
5130 self.minimumreplication = adapter_args.get('minimumreplication',0)
5131
5132
5133
5134
5135 self.safe = adapter_args.get('safe',True)
5136
5137 if isinstance(m,tuple):
5138 m = {"database" : m[1]}
5139 if m.get('database')==None:
5140 raise SyntaxError("Database is required!")
5141 def connector(uri=self.uri,m=m):
5142 try:
5143
5144 if hasattr(self.driver, "MongoClient"):
5145 Connection = self.driver.MongoClient
5146 else:
5147 Connection = self.driver.Connection
5148 return Connection(uri)[m.get('database')]
5149 except self.driver.errors.ConnectionFailure:
5150 inst = sys.exc_info()[1]
5151 raise SyntaxError("The connection to " +
5152 uri + " could not be made")
5153
5154 self.reconnect(connector,cursor=False)
5155
5157 """ Convert input to a valid Mongodb ObjectId instance
5158
5159 self.object_id("<random>") -> ObjectId (not unique) instance """
5160 if not arg:
5161 arg = 0
5162 if isinstance(arg, basestring):
5163
5164 rawhex = len(arg.replace("0x", "").replace("L", "")) == 24
5165 if arg.isdigit() and (not rawhex):
5166 arg = int(arg)
5167 elif arg == "<random>":
5168 arg = int("0x%sL" % \
5169 "".join([self.random.choice("0123456789abcdef") \
5170 for x in range(24)]), 0)
5171 elif arg.isalnum():
5172 if not arg.startswith("0x"):
5173 arg = "0x%s" % arg
5174 try:
5175 arg = int(arg, 0)
5176 except ValueError, e:
5177 raise ValueError(
5178 "invalid objectid argument string: %s" % e)
5179 else:
5180 raise ValueError("Invalid objectid argument string. " +
5181 "Requires an integer or base 16 value")
5182 elif isinstance(arg, self.ObjectId):
5183 return arg
5184 if not isinstance(arg, (int, long)):
5185 raise TypeError("object_id argument must be of type " +
5186 "ObjectId or an objectid representable integer")
5187 if arg == 0:
5188 hexvalue = "".zfill(24)
5189 else:
5190 hexvalue = hex(arg)[2:].replace("L", "")
5191 return self.ObjectId(hexvalue)
5192
5194 value = NoSQLAdapter.represent(self, obj, fieldtype)
5195 if fieldtype =='date':
5196 if value == None:
5197 return value
5198
5199 t = datetime.time(0, 0, 0)
5200
5201
5202 return datetime.datetime.combine(value, t)
5203 elif fieldtype == 'time':
5204 if value == None:
5205 return value
5206
5207 d = datetime.date(2000, 1, 1)
5208
5209
5210 return datetime.datetime.combine(d, value)
5211 elif fieldtype == 'list:string' or \
5212 fieldtype == 'list:integer' or \
5213 fieldtype == 'list:reference':
5214 return value
5215 return value
5216
5217
5218
5219
5220 - def insert(self, table, fields, safe=None):
5221 if safe==None:
5222 safe = self.safe
5223 ctable = self.connection[table._tablename]
5224 values = dict()
5225 for k, v in fields:
5226 if not k.name in ["id", "safe"]:
5227 fieldname = k.name
5228 fieldtype = table[k.name].type
5229 if ("reference" in fieldtype) or (fieldtype=="id"):
5230 values[fieldname] = self.object_id(v)
5231 else:
5232 values[fieldname] = self.represent(v, fieldtype)
5233 ctable.insert(values, safe=safe)
5234 return int(str(values['_id']), 16)
5235
5236 - def create_table(self, table, migrate=True, fake_migrate=False,
5237 polymodel=None, isCapped=False):
5238 if isCapped:
5239 raise RuntimeError("Not implemented")
5240
5241 - def count(self, query, distinct=None, snapshot=True):
5242 if distinct:
5243 raise RuntimeError("COUNT DISTINCT not supported")
5244 if not isinstance(query,Query):
5245 raise SyntaxError("Not Supported")
5246 tablename = self.get_table(query)
5247 return int(self.select(query,[self.db[tablename]._id], {},
5248 count=True,snapshot=snapshot)['count'])
5249
5250
5251
5252
5253
5254 - def expand(self, expression, field_type=None):
5255 if isinstance(expression, Query):
5256
5257
5258
5259
5260
5261 if isinstance(expression.first,Field) and \
5262 ((expression.first.type == 'id') or \
5263 ("reference" in expression.first.type)):
5264 if expression.first.type == 'id':
5265 expression.first.name = '_id'
5266
5267 expression.second = self.object_id(expression.second)
5268 result = expression.op(expression.first, expression.second)
5269 if isinstance(expression, Field):
5270 if expression.type=='id':
5271 result = "_id"
5272 else:
5273 result = expression.name
5274
5275 elif isinstance(expression, (Expression, Query)):
5276 if not expression.second is None:
5277 result = expression.op(expression.first, expression.second)
5278 elif not expression.first is None:
5279 result = expression.op(expression.first)
5280 elif not isinstance(expression.op, str):
5281 result = expression.op()
5282 else:
5283 result = expression.op
5284 elif field_type:
5285 result = str(self.represent(expression,field_type))
5286 elif isinstance(expression,(list,tuple)):
5287 result = ','.join(self.represent(item,field_type) for
5288 item in expression)
5289 else:
5290 result = expression
5291 return result
5292
5293 - def _select(self, query, fields, attributes):
5294 if 'for_update' in attributes:
5295 logging.warn('mongodb does not support for_update')
5296 for key in set(attributes.keys())-set(('limitby',
5297 'orderby','for_update')):
5298 if attributes[key]!=None:
5299 logging.warn('select attribute not implemented: %s' % key)
5300
5301 new_fields=[]
5302 mongosort_list = []
5303
5304
5305 orderby = attributes.get('orderby', False)
5306 limitby = attributes.get('limitby', False)
5307
5308 if orderby:
5309 if isinstance(orderby, (list, tuple)):
5310 orderby = xorify(orderby)
5311
5312
5313 for f in self.expand(orderby).split(','):
5314 if f.startswith('-'):
5315 mongosort_list.append((f[1:], -1))
5316 else:
5317 mongosort_list.append((f, 1))
5318
5319 if limitby:
5320 limitby_skip, limitby_limit = limitby
5321 else:
5322 limitby_skip = limitby_limit = 0
5323
5324 mongofields_dict = self.SON()
5325 mongoqry_dict = {}
5326 for item in fields:
5327 if isinstance(item, SQLALL):
5328 new_fields += item._table
5329 else:
5330 new_fields.append(item)
5331 fields = new_fields
5332 if isinstance(query,Query):
5333 tablename = self.get_table(query)
5334 elif len(fields) != 0:
5335 tablename = fields[0].tablename
5336 else:
5337 raise SyntaxError("The table name could not be found in " +
5338 "the query nor from the select statement.")
5339
5340 mongoqry_dict = self.expand(query)
5341 fields = fields or self.db[tablename]
5342 for field in fields:
5343 mongofields_dict[field.name] = 1
5344
5345 return tablename, mongoqry_dict, mongofields_dict, mongosort_list, \
5346 limitby_limit, limitby_skip
5347
5348
5349 - def select(self, query, fields, attributes, count=False,
5350 snapshot=False):
5351
5352 tablename, mongoqry_dict, mongofields_dict, mongosort_list, \
5353 limitby_limit, limitby_skip = self._select(query, fields, attributes)
5354 ctable = self.connection[tablename]
5355
5356 if count:
5357 return {'count' : ctable.find(
5358 mongoqry_dict, mongofields_dict,
5359 skip=limitby_skip, limit=limitby_limit,
5360 sort=mongosort_list, snapshot=snapshot).count()}
5361 else:
5362
5363 mongo_list_dicts = ctable.find(mongoqry_dict,
5364 mongofields_dict, skip=limitby_skip,
5365 limit=limitby_limit, sort=mongosort_list,
5366 snapshot=snapshot)
5367 rows = []
5368
5369
5370 colnames = []
5371 newnames = []
5372 for field in fields:
5373 colname = str(field)
5374 colnames.append(colname)
5375 tablename, fieldname = colname.split(".")
5376 if fieldname == "_id":
5377
5378 field.name = "id"
5379 newnames.append(".".join((tablename, field.name)))
5380
5381 for record in mongo_list_dicts:
5382 row=[]
5383 for colname in colnames:
5384 tablename, fieldname = colname.split(".")
5385
5386
5387 if fieldname == "id": fieldname = "_id"
5388 if fieldname in record:
5389 if isinstance(record[fieldname],
5390 self.ObjectId):
5391 value = int(str(record[fieldname]), 16)
5392 else:
5393 value = record[fieldname]
5394 else:
5395 value = None
5396 row.append(value)
5397 rows.append(row)
5398 processor = attributes.get('processor', self.parse)
5399 result = processor(rows, fields, newnames, False)
5400 return result
5401
5402
5406
5407 - def drop(self, table, mode=''):
5410
5411
5412 - def truncate(self, table, mode, safe=None):
5413 if safe == None:
5414 safe=self.safe
5415 ctable = self.connection[table._tablename]
5416 ctable.remove(None, safe=True)
5417
5418 - def oupdate(self, tablename, query, fields):
5419 if not isinstance(query, Query):
5420 raise SyntaxError("Not Supported")
5421 filter = None
5422 if query:
5423 filter = self.expand(query)
5424 modify = {'$set': dict((k.name, self.represent(v, k.type)) for
5425 k, v in fields)}
5426 return modify, filter
5427
5428 - def update(self, tablename, query, fields, safe=None):
5429 if safe == None:
5430 safe = self.safe
5431
5432
5433 if not isinstance(query, Query):
5434 raise RuntimeError("Not implemented")
5435 amount = self.count(query, False)
5436 modify, filter = self.oupdate(tablename, query, fields)
5437 try:
5438 result = self.connection[tablename].update(filter,
5439 modify, multi=True, safe=safe)
5440 if safe:
5441 try:
5442
5443 return result["n"]
5444 except (KeyError, AttributeError, TypeError):
5445 return amount
5446 else:
5447 return amount
5448 except Exception, e:
5449
5450 raise RuntimeError("uncaught exception when updating rows: %s" % e)
5451
5452
5453 - def _update(self,tablename,query,fields):
5455
5456 - def delete(self, tablename, query, safe=None):
5457 if safe is None:
5458 safe = self.safe
5459 amount = 0
5460 amount = self.count(query, False)
5461 if not isinstance(query, Query):
5462 raise RuntimeError("query type %s is not supported" % \
5463 type(query))
5464 filter = self.expand(query)
5465 self._delete(tablename, filter, safe=safe)
5466 return amount
5467
5468 - def _delete(self, tablename, filter, safe=None):
5469 return self.connection[tablename].remove(filter, safe=safe)
5470
5472 return [self.insert(table,item) for item in items]
5473
5474
5475 - def NOT(self, first):
5476 result = {}
5477 result["$not"] = self.expand(first)
5478 return result
5479
5480 - def AND(self,first,second):
5485
5486 - def OR(self,first,second):
5487
5488 result = {}
5489 f = self.expand(first)
5490 s = self.expand(second)
5491 result['$or'] = [f,s]
5492 return result
5493
5494 - def BELONGS(self, first, second):
5495 if isinstance(second, str):
5496 return {self.expand(first) : {"$in" : [ second[:-1]]} }
5497 elif second==[] or second==():
5498 return {1:0}
5499 items = [self.expand(item, first.type) for item in second]
5500 return {self.expand(first) : {"$in" : items} }
5501
5502 - def EQ(self,first,second):
5503 result = {}
5504 result[self.expand(first)] = self.expand(second)
5505 return result
5506
5507 - def NE(self, first, second=None):
5508 result = {}
5509 result[self.expand(first)] = {'$ne': self.expand(second)}
5510 return result
5511
5512 - def LT(self,first,second=None):
5513 if second is None:
5514 raise RuntimeError("Cannot compare %s < None" % first)
5515 result = {}
5516 result[self.expand(first)] = {'$lt': self.expand(second)}
5517 return result
5518
5519 - def LE(self,first,second=None):
5520 if second is None:
5521 raise RuntimeError("Cannot compare %s <= None" % first)
5522 result = {}
5523 result[self.expand(first)] = {'$lte': self.expand(second)}
5524 return result
5525
5526 - def GT(self,first,second):
5527 result = {}
5528 result[self.expand(first)] = {'$gt': self.expand(second)}
5529 return result
5530
5531 - def GE(self,first,second=None):
5532 if second is None:
5533 raise RuntimeError("Cannot compare %s >= None" % first)
5534 result = {}
5535 result[self.expand(first)] = {'$gte': self.expand(second)}
5536 return result
5537
5538 - def ADD(self, first, second):
5542
5543 - def SUB(self, first, second):
5547
5548 - def MUL(self, first, second):
5552
5553 - def DIV(self, first, second):
5557
5558 - def MOD(self, first, second):
5562
5563 - def AS(self, first, second):
5564 raise NotImplementedError(self.error_messages["javascript_needed"])
5565 return '%s AS %s' % (self.expand(first), second)
5566
5567
5568
5569
5570 - def ON(self, first, second):
5571 raise NotImplementedError("This is not possible in NoSQL" +
5572 " but can be simulated with a wrapper.")
5573 return '%s ON %s' % (self.expand(first), self.expand(second))
5574
5575
5576
5577
5578 - def COMMA(self, first, second):
5580
5581 - def LIKE(self, first, second):
5585
5587
5588 return {self.expand(first): ('/^%s/' % \
5589 self.expand(second, 'string'))}
5590
5592
5593 return {self.expand(first): ('/%s^/' % \
5594 self.expand(second, 'string'))}
5595
5596 - def CONTAINS(self, first, second, case_sensitive=False):
5597
5598
5599
5600 return {self.expand(first) : ('/%s/' % \
5601 self.expand(second, 'string'))}
5602
5603 - def LIKE(self, first, second):
5608
5609
5611
5612 import re
5613 return {self.expand(first): {'$regex' : '^' +
5614 re.escape(self.expand(second,
5615 'string'))}}
5616
5617
5619
5620
5621
5622
5623 import re
5624 return {self.expand(first): {'$regex': \
5625 re.escape(self.expand(second, 'string')) + '$'}}
5626
5627
5628 - def CONTAINS(self, first, second, case_sensitive=False):
5629
5630
5631
5632
5633 return {self.expand(first) : {' $regex': \
5634 ".*" + re.escape(self.expand(second, 'string')) + ".*"}}
5635
5638 drivers = ('imaplib',)
5639
5640 """ IMAP server adapter
5641
5642 This class is intended as an interface with
5643 email IMAP servers to perform simple queries in the
5644 web2py DAL query syntax, so email read, search and
5645 other related IMAP mail services (as those implemented
5646 by brands like Google(r), and Yahoo!(r)
5647 can be managed from web2py applications.
5648
5649 The code uses examples by Yuji Tomita on this post:
5650 http://yuji.wordpress.com/2011/06/22/python-imaplib-imap-example-with-gmail/#comment-1137
5651 and is based in docs for Python imaplib, python email
5652 and email IETF's (i.e. RFC2060 and RFC3501)
5653
5654 This adapter was tested with a small set of operations with Gmail(r). Other
5655 services requests could raise command syntax and response data issues.
5656
5657 It creates its table and field names "statically",
5658 meaning that the developer should leave the table and field
5659 definitions to the DAL instance by calling the adapter's
5660 .define_tables() method. The tables are defined with the
5661 IMAP server mailbox list information.
5662
5663 .define_tables() returns a dictionary mapping dal tablenames
5664 to the server mailbox names with the following structure:
5665
5666 {<tablename>: str <server mailbox name>}
5667
5668 Here is a list of supported fields:
5669
5670 Field Type Description
5671 ################################################################
5672 uid string
5673 answered boolean Flag
5674 created date
5675 content list:string A list of text or html parts
5676 to string
5677 cc string
5678 bcc string
5679 size integer the amount of octets of the message*
5680 deleted boolean Flag
5681 draft boolean Flag
5682 flagged boolean Flag
5683 sender string
5684 recent boolean Flag
5685 seen boolean Flag
5686 subject string
5687 mime string The mime header declaration
5688 email string The complete RFC822 message**
5689 attachments <type list> Each non text part as dict
5690 encoding string The main detected encoding
5691
5692 *At the application side it is measured as the length of the RFC822
5693 message string
5694
5695 WARNING: As row id's are mapped to email sequence numbers,
5696 make sure your imap client web2py app does not delete messages
5697 during select or update actions, to prevent
5698 updating or deleting different messages.
5699 Sequence numbers change whenever the mailbox is updated.
5700 To avoid this sequence numbers issues, it is recommended the use
5701 of uid fields in query references (although the update and delete
5702 in separate actions rule still applies).
5703
5704 # This is the code recommended to start imap support
5705 # at the app's model:
5706
5707 imapdb = DAL("imap://user:password@server:port", pool_size=1) # port 993 for ssl
5708 imapdb.define_tables()
5709
5710 Here is an (incomplete) list of possible imap commands:
5711
5712 # Count today's unseen messages
5713 # smaller than 6000 octets from the
5714 # inbox mailbox
5715
5716 q = imapdb.INBOX.seen == False
5717 q &= imapdb.INBOX.created == datetime.date.today()
5718 q &= imapdb.INBOX.size < 6000
5719 unread = imapdb(q).count()
5720
5721 # Fetch last query messages
5722 rows = imapdb(q).select()
5723
5724 # it is also possible to filter query select results with limitby and
5725 # sequences of mailbox fields
5726
5727 set.select(<fields sequence>, limitby=(<int>, <int>))
5728
5729 # Mark last query messages as seen
5730 messages = [row.uid for row in rows]
5731 seen = imapdb(imapdb.INBOX.uid.belongs(messages)).update(seen=True)
5732
5733 # Delete messages in the imap database that have mails from mr. Gumby
5734
5735 deleted = 0
5736 for mailbox in imapdb.tables
5737 deleted += imapdb(imapdb[mailbox].sender.contains("gumby")).delete()
5738
5739 # It is possible also to mark messages for deletion instead of ereasing them
5740 # directly with set.update(deleted=True)
5741
5742
5743 # This object give access
5744 # to the adapter auto mailbox
5745 # mapped names (which native
5746 # mailbox has what table name)
5747
5748 imapdb.mailboxes <dict> # tablename, server native name pairs
5749
5750 # To retrieve a table native mailbox name use:
5751 imapdb.<table>.mailbox
5752
5753 ### New features v2.4.1:
5754
5755 # Declare mailboxes statically with tablename, name pairs
5756 # This avoids the extra server names retrieval
5757
5758 imapdb.define_tables({"inbox": "INBOX"})
5759
5760 # Selects without content/attachments/email columns will only
5761 # fetch header and flags
5762
5763 imapdb(q).select(imapdb.INBOX.sender, imapdb.INBOX.subject)
5764 """
5765
5766 types = {
5767 'string': str,
5768 'text': str,
5769 'date': datetime.date,
5770 'datetime': datetime.datetime,
5771 'id': long,
5772 'boolean': bool,
5773 'integer': int,
5774 'bigint': long,
5775 'blob': str,
5776 'list:string': str,
5777 }
5778
5779 dbengine = 'imap'
5780
5781 REGEX_URI = re.compile('^(?P<user>[^:]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?$')
5782
5783 - def __init__(self,
5784 db,
5785 uri,
5786 pool_size=0,
5787 folder=None,
5788 db_codec ='UTF-8',
5789 credential_decoder=IDENTITY,
5790 driver_args={},
5791 adapter_args={},
5792 do_connect=True,
5793 after_connection=None):
5794
5795
5796
5797
5798 self.db = db
5799 self.uri = uri
5800 if do_connect: self.find_driver(adapter_args)
5801 self.pool_size=pool_size
5802 self.folder = folder
5803 self.db_codec = db_codec
5804 self._after_connection = after_connection
5805 self.credential_decoder = credential_decoder
5806 self.driver_args = driver_args
5807 self.adapter_args = adapter_args
5808 self.mailbox_size = None
5809 self.static_names = None
5810 self.charset = sys.getfilesystemencoding()
5811
5812 self.imap4 = None
5813 uri = uri.split("://")[1]
5814
5815 """ MESSAGE is an identifier for sequence number"""
5816
5817 self.flags = ['\\Deleted', '\\Draft', '\\Flagged',
5818 '\\Recent', '\\Seen', '\\Answered']
5819 self.search_fields = {
5820 'id': 'MESSAGE', 'created': 'DATE',
5821 'uid': 'UID', 'sender': 'FROM',
5822 'to': 'TO', 'cc': 'CC',
5823 'bcc': 'BCC', 'content': 'TEXT',
5824 'size': 'SIZE', 'deleted': '\\Deleted',
5825 'draft': '\\Draft', 'flagged': '\\Flagged',
5826 'recent': '\\Recent', 'seen': '\\Seen',
5827 'subject': 'SUBJECT', 'answered': '\\Answered',
5828 'mime': None, 'email': None,
5829 'attachments': None
5830 }
5831
5832 db['_lastsql'] = ''
5833
5834 m = self.REGEX_URI.match(uri)
5835 user = m.group('user')
5836 password = m.group('password')
5837 host = m.group('host')
5838 port = int(m.group('port'))
5839 over_ssl = False
5840 if port==993:
5841 over_ssl = True
5842
5843 driver_args.update(host=host,port=port, password=password, user=user)
5844 def connector(driver_args=driver_args):
5845
5846
5847 if over_ssl:
5848 self.imap4 = self.driver.IMAP4_SSL
5849 else:
5850 self.imap4 = self.driver.IMAP4
5851 connection = self.imap4(driver_args["host"], driver_args["port"])
5852 data = connection.login(driver_args["user"], driver_args["password"])
5853
5854
5855 connection.mailbox_names = None
5856
5857
5858 connection.cursor = lambda : True
5859
5860 return connection
5861
5862 self.db.define_tables = self.define_tables
5863 self.connector = connector
5864 if do_connect: self.reconnect()
5865
5910
5912 last_message = None
5913
5914
5915 if not isinstance(self.connection.mailbox_names, dict):
5916 self.get_mailboxes()
5917 try:
5918 result = self.connection.select(self.connection.mailbox_names[tablename])
5919 last_message = int(result[1][0])
5920 except (IndexError, ValueError, TypeError, KeyError):
5921 e = sys.exc_info()[1]
5922 LOGGER.debug("Error retrieving the last mailbox sequence number. %s" % str(e))
5923 return last_message
5924
5926 if not isinstance(self.connection.mailbox_names, dict):
5927 self.get_mailboxes()
5928
5929
5930 last_message = self.get_last_message(tablename)
5931 result, data = self.connection.uid("search", None, "(ALL)")
5932 uid_list = data[0].strip().split()
5933 if len(uid_list) <= 0:
5934 return None
5935 else:
5936 return (uid_list[0], uid_list[-1])
5937
5939 if add is None:
5940 add = datetime.timedelta()
5941 """ Convert a date object to a string
5942 with d-Mon-Y style for IMAP or the inverse
5943 case
5944
5945 add <timedelta> adds to the date object
5946 """
5947 months = [None, "Jan","Feb","Mar","Apr","May","Jun",
5948 "Jul", "Aug","Sep","Oct","Nov","Dec"]
5949 if isinstance(date, basestring):
5950
5951 try:
5952 dayname, datestring = date.split(",")
5953 except (ValueError):
5954 LOGGER.debug("Could not parse date text: %s" % date)
5955 return None
5956 date_list = datestring.strip().split()
5957 year = int(date_list[2])
5958 month = months.index(date_list[1])
5959 day = int(date_list[0])
5960 hms = map(int, date_list[3].split(":"))
5961 return datetime.datetime(year, month, day,
5962 hms[0], hms[1], hms[2]) + add
5963 elif isinstance(date, (datetime.datetime, datetime.date)):
5964 return (date + add).strftime("%d-%b-%Y")
5965
5966 else:
5967 return None
5968
5969 @staticmethod
5971 from email.header import decode_header
5972 text, encoding = decode_header(f)[0]
5973 return text
5974
5975 - def encode_text(self, text, charset, errors="replace"):
5976 """ convert text for mail to unicode"""
5977 if text is None:
5978 text = ""
5979 else:
5980 if isinstance(text, str):
5981 if charset is None:
5982 text = unicode(text, "utf-8", errors)
5983 else:
5984 text = unicode(text, charset, errors)
5985 else:
5986 raise Exception("Unsupported mail text type %s" % type(text))
5987 return text.encode("utf-8")
5988
5990 charset = message.get_content_charset()
5991 return charset
5992
5994 """ Query the mail database for mailbox names """
5995 if self.static_names:
5996
5997 self.connection.mailbox_names = self.static_names
5998 return self.static_names.keys()
5999
6000 mailboxes_list = self.connection.list()
6001 self.connection.mailbox_names = dict()
6002 mailboxes = list()
6003 x = 0
6004 for item in mailboxes_list[1]:
6005 x = x + 1
6006 item = item.strip()
6007 if not "NOSELECT" in item.upper():
6008 sub_items = item.split("\"")
6009 sub_items = [sub_item for sub_item in sub_items \
6010 if len(sub_item.strip()) > 0]
6011
6012 mailbox = sub_items[-1]
6013
6014
6015 mailbox_name = re.sub('^[_0-9]*', '', re.sub('[^_\w]','',re.sub('[/ ]','_',mailbox)))
6016 mailboxes.append(mailbox_name)
6017 self.connection.mailbox_names[mailbox_name] = mailbox
6018
6019 return mailboxes
6020
6022 nofield = True
6023 tablename = None
6024 attr = query
6025 while nofield:
6026 if hasattr(attr, "first"):
6027 attr = attr.first
6028 if isinstance(attr, Field):
6029 return attr.tablename
6030 elif isinstance(attr, Query):
6031 pass
6032 else:
6033 return None
6034 else:
6035 return None
6036 return tablename
6037
6039 if self.search_fields.get(flag, None) in self.flags:
6040 return True
6041 else:
6042 return False
6043
6045 """
6046 Auto create common IMAP fileds
6047
6048 This function creates fields definitions "statically"
6049 meaning that custom fields as in other adapters should
6050 not be supported and definitions handled on a service/mode
6051 basis (local syntax for Gmail(r), Ymail(r)
6052
6053 Returns a dictionary with tablename, server native mailbox name
6054 pairs.
6055 """
6056 if mailbox_names:
6057
6058 self.static_names = mailbox_names
6059 else:
6060 self.static_names = None
6061 if not isinstance(self.connection.mailbox_names, dict):
6062 self.get_mailboxes()
6063
6064 names = self.connection.mailbox_names.keys()
6065
6066 for name in names:
6067 self.db.define_table("%s" % name,
6068 Field("uid", "string", writable=False),
6069 Field("answered", "boolean"),
6070 Field("created", "datetime", writable=False),
6071 Field("content", "list:string", writable=False),
6072 Field("to", "string", writable=False),
6073 Field("cc", "string", writable=False),
6074 Field("bcc", "string", writable=False),
6075 Field("size", "integer", writable=False),
6076 Field("deleted", "boolean"),
6077 Field("draft", "boolean"),
6078 Field("flagged", "boolean"),
6079 Field("sender", "string", writable=False),
6080 Field("recent", "boolean", writable=False),
6081 Field("seen", "boolean"),
6082 Field("subject", "string", writable=False),
6083 Field("mime", "string", writable=False),
6084 Field("email", "string", writable=False, readable=False),
6085 Field("attachments", list, writable=False, readable=False),
6086 Field("encoding")
6087 )
6088
6089
6090
6091 self.db[name].mailbox = \
6092 self.connection.mailbox_names[name]
6093
6094
6095 self.db[name].to.represent = self.db[name].cc.represent = \
6096 self.db[name].bcc.represent = self.db[name].sender.represent = \
6097 self.db[name].subject.represent = self.header_represent
6098
6099
6100 self.db.mailboxes = self.connection.mailbox_names
6101 return self.db.mailboxes
6102
6107
6108 - def _select(self, query, fields, attributes):
6112
6113 - def select(self, query, fields, attributes):
6114 """ Search and Fetch records and return web2py rows
6115 """
6116
6117 if use_common_filters(query):
6118 query = self.common_filter(query, [self.get_query_mailbox(query),])
6119
6120 import email
6121
6122
6123 tablename = None
6124 fetch_results = list()
6125
6126 if isinstance(query, Query):
6127 tablename = self.get_table(query)
6128 mailbox = self.connection.mailbox_names.get(tablename, None)
6129 if mailbox is None:
6130 raise ValueError("Mailbox name not found: %s" % mailbox)
6131 else:
6132
6133 result, selected = self.connection.select(mailbox, True)
6134 if result != "OK":
6135 raise Exception("IMAP error: %s" % selected)
6136 self.mailbox_size = int(selected[0])
6137 search_query = "(%s)" % str(query).strip()
6138 search_result = self.connection.uid("search", None, search_query)
6139
6140 if search_result[0] == "OK":
6141
6142
6143
6144
6145 limitby = attributes.get('limitby', None)
6146 messages_set = search_result[1][0].split()
6147
6148 messages_set.reverse()
6149 if limitby is not None:
6150
6151 messages_set = messages_set[int(limitby[0]):int(limitby[1])]
6152
6153
6154 if any([(field.name in ["content", "size",
6155 "attachments", "email"]) for
6156 field in fields]):
6157 imap_fields = "(RFC822 FLAGS)"
6158 else:
6159 imap_fields = "(RFC822.HEADER FLAGS)"
6160
6161 if len(messages_set) > 0:
6162
6163
6164
6165
6166 for uid in messages_set:
6167
6168 typ, data = self.connection.uid("fetch", uid, imap_fields)
6169 if typ == "OK":
6170 fr = {"message": int(data[0][0].split()[0]),
6171 "uid": int(uid),
6172 "email": email.message_from_string(data[0][1]),
6173 "raw_message": data[0][1]}
6174 fr["multipart"] = fr["email"].is_multipart()
6175
6176 fr["flags"] = self.driver.ParseFlags(data[1])
6177 fetch_results.append(fr)
6178 else:
6179
6180 raise Exception("IMAP error retrieving the body: %s" % data)
6181 else:
6182 raise Exception("IMAP search error: %s" % search_result[1])
6183 elif isinstance(query, (Expression, basestring)):
6184 raise NotImplementedError()
6185 else:
6186 raise TypeError("Unexpected query type")
6187
6188 imapqry_dict = {}
6189 imapfields_dict = {}
6190
6191 if len(fields) == 1 and isinstance(fields[0], SQLALL):
6192 allfields = True
6193 elif len(fields) == 0:
6194 allfields = True
6195 else:
6196 allfields = False
6197 if allfields:
6198 colnames = ["%s.%s" % (tablename, field) for field in self.search_fields.keys()]
6199 else:
6200 colnames = ["%s.%s" % (tablename, field.name) for field in fields]
6201
6202 for k in colnames:
6203 imapfields_dict[k] = k
6204
6205 imapqry_list = list()
6206 imapqry_array = list()
6207 for fr in fetch_results:
6208 attachments = []
6209 content = []
6210 size = 0
6211 n = int(fr["message"])
6212 item_dict = dict()
6213 message = fr["email"]
6214 uid = fr["uid"]
6215 charset = self.get_charset(message)
6216 flags = fr["flags"]
6217 raw_message = fr["raw_message"]
6218
6219
6220
6221
6222
6223
6224
6225
6226
6227
6228 if "%s.id" % tablename in colnames:
6229 item_dict["%s.id" % tablename] = n
6230 if "%s.created" % tablename in colnames:
6231 item_dict["%s.created" % tablename] = self.convert_date(message["Date"])
6232 if "%s.uid" % tablename in colnames:
6233 item_dict["%s.uid" % tablename] = uid
6234 if "%s.sender" % tablename in colnames:
6235
6236
6237
6238 item_dict["%s.sender" % tablename] = message["From"]
6239 if "%s.to" % tablename in colnames:
6240 item_dict["%s.to" % tablename] = message["To"]
6241 if "%s.cc" % tablename in colnames:
6242 if "Cc" in message.keys():
6243 item_dict["%s.cc" % tablename] = message["Cc"]
6244 else:
6245 item_dict["%s.cc" % tablename] = ""
6246 if "%s.bcc" % tablename in colnames:
6247 if "Bcc" in message.keys():
6248 item_dict["%s.bcc" % tablename] = message["Bcc"]
6249 else:
6250 item_dict["%s.bcc" % tablename] = ""
6251 if "%s.deleted" % tablename in colnames:
6252 item_dict["%s.deleted" % tablename] = "\\Deleted" in flags
6253 if "%s.draft" % tablename in colnames:
6254 item_dict["%s.draft" % tablename] = "\\Draft" in flags
6255 if "%s.flagged" % tablename in colnames:
6256 item_dict["%s.flagged" % tablename] = "\\Flagged" in flags
6257 if "%s.recent" % tablename in colnames:
6258 item_dict["%s.recent" % tablename] = "\\Recent" in flags
6259 if "%s.seen" % tablename in colnames:
6260 item_dict["%s.seen" % tablename] = "\\Seen" in flags
6261 if "%s.subject" % tablename in colnames:
6262 item_dict["%s.subject" % tablename] = message["Subject"]
6263 if "%s.answered" % tablename in colnames:
6264 item_dict["%s.answered" % tablename] = "\\Answered" in flags
6265 if "%s.mime" % tablename in colnames:
6266 item_dict["%s.mime" % tablename] = message.get_content_type()
6267 if "%s.encoding" % tablename in colnames:
6268 item_dict["%s.encoding" % tablename] = charset
6269
6270
6271
6272
6273
6274
6275 if "%s.email" % tablename in colnames:
6276
6277 item_dict["%s.email" % tablename] = raw_message
6278
6279
6280
6281
6282
6283
6284 for part in message.walk():
6285 maintype = part.get_content_maintype()
6286 if ("%s.attachments" % tablename in colnames) or \
6287 ("%s.content" % tablename in colnames):
6288 if "%s.attachments" % tablename in colnames:
6289 if not ("text" in maintype):
6290 payload = part.get_payload(decode=True)
6291 if payload:
6292 attachment = {
6293 "payload": payload,
6294 "filename": part.get_filename(),
6295 "encoding": part.get_content_charset(),
6296 "mime": part.get_content_type(),
6297 "disposition": part["Content-Disposition"]}
6298 attachments.append(attachment)
6299 if "%s.content" % tablename in colnames:
6300 payload = part.get_payload(decode=True)
6301 part_charset = self.get_charset(part)
6302 if "text" in maintype:
6303 if payload:
6304 content.append(self.encode_text(payload, part_charset))
6305 if "%s.size" % tablename in colnames:
6306 if part is not None:
6307 size += len(str(part))
6308 item_dict["%s.content" % tablename] = bar_encode(content)
6309 item_dict["%s.attachments" % tablename] = attachments
6310 item_dict["%s.size" % tablename] = size
6311 imapqry_list.append(item_dict)
6312
6313
6314
6315 for item_dict in imapqry_list:
6316 imapqry_array_item = list()
6317 for fieldname in colnames:
6318 imapqry_array_item.append(item_dict[fieldname])
6319 imapqry_array.append(imapqry_array_item)
6320
6321
6322 colnames = colnames
6323 processor = attributes.get('processor',self.parse)
6324 return processor(imapqry_array, fields, colnames)
6325
6326 - def _update(self, tablename, query, fields, commit=False):
6327
6328 commands = list()
6329 if use_common_filters(query):
6330 query = self.common_filter(query, [tablename,])
6331 mark = []
6332 unmark = []
6333 if query:
6334 for item in fields:
6335 field = item[0]
6336 name = field.name
6337 value = item[1]
6338 if self.is_flag(name):
6339 flag = self.search_fields[name]
6340 if (value is not None) and (flag != "\\Recent"):
6341 if value:
6342 mark.append(flag)
6343 else:
6344 unmark.append(flag)
6345 result, data = self.connection.select(
6346 self.connection.mailbox_names[tablename])
6347 string_query = "(%s)" % query
6348 result, data = self.connection.search(None, string_query)
6349 store_list = [item.strip() for item in data[0].split()
6350 if item.strip().isdigit()]
6351
6352 for number in store_list:
6353 result = None
6354 if len(mark) > 0:
6355 commands.append((number, "+FLAGS", "(%s)" % " ".join(mark)))
6356 if len(unmark) > 0:
6357 commands.append((number, "-FLAGS", "(%s)" % " ".join(unmark)))
6358 return commands
6359
6360 - def update(self, tablename, query, fields):
6361 rowcount = 0
6362 commands = self._update(tablename, query, fields)
6363 for command in commands:
6364 result, data = self.connection.store(*command)
6365 if result == "OK":
6366 rowcount += 1
6367 else:
6368 raise Exception("IMAP storing error: %s" % data)
6369 return rowcount
6370
6371 - def _count(self, query, distinct=None):
6372 raise NotImplementedError()
6373
6374 - def count(self,query,distinct=None):
6386
6387 - def delete(self, tablename, query):
6388 counter = 0
6389 if query:
6390 if use_common_filters(query):
6391 query = self.common_filter(query, [tablename,])
6392 result, data = self.connection.select(self.connection.mailbox_names[tablename])
6393 string_query = "(%s)" % query
6394 result, data = self.connection.search(None, string_query)
6395 store_list = [item.strip() for item in data[0].split() if item.strip().isdigit()]
6396 for number in store_list:
6397 result, data = self.connection.store(number, "+FLAGS", "(\\Deleted)")
6398 if result == "OK":
6399 counter += 1
6400 else:
6401 raise Exception("IMAP store error: %s" % data)
6402 if counter > 0:
6403 result, data = self.connection.expunge()
6404 return counter
6405
6406 - def BELONGS(self, first, second):
6407 result = None
6408 name = self.search_fields[first.name]
6409 if name == "MESSAGE":
6410 values = [str(val) for val in second if str(val).isdigit()]
6411 result = "%s" % ",".join(values).strip()
6412
6413 elif name == "UID":
6414 values = [str(val) for val in second if str(val).isdigit()]
6415 result = "UID %s" % ",".join(values).strip()
6416
6417 else:
6418 raise Exception("Operation not supported")
6419
6420 return result
6421
6422 - def CONTAINS(self, first, second, case_sensitive=False):
6423
6424 result = None
6425 name = self.search_fields[first.name]
6426
6427 if name in ("FROM", "TO", "SUBJECT", "TEXT"):
6428 result = "%s \"%s\"" % (name, self.expand(second))
6429 else:
6430 if first.name in ("cc", "bcc"):
6431 result = "%s \"%s\"" % (first.name.upper(), self.expand(second))
6432 elif first.name == "mime":
6433 result = "HEADER Content-Type \"%s\"" % self.expand(second)
6434 else:
6435 raise Exception("Operation not supported")
6436 return result
6437
6438 - def GT(self, first, second):
6439 result = None
6440 name = self.search_fields[first.name]
6441 if name == "MESSAGE":
6442 last_message = self.get_last_message(first.tablename)
6443 result = "%d:%d" % (int(self.expand(second)) + 1, last_message)
6444 elif name == "UID":
6445
6446
6447
6448 try:
6449 pedestal, threshold = self.get_uid_bounds(first.tablename)
6450 except TypeError:
6451 e = sys.exc_info()[1]
6452 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6453 return ""
6454 try:
6455 lower_limit = int(self.expand(second)) + 1
6456 except (ValueError, TypeError):
6457 e = sys.exc_info()[1]
6458 raise Exception("Operation not supported (non integer UID)")
6459 result = "UID %s:%s" % (lower_limit, threshold)
6460 elif name == "DATE":
6461 result = "SINCE %s" % self.convert_date(second, add=datetime.timedelta(1))
6462 elif name == "SIZE":
6463 result = "LARGER %s" % self.expand(second)
6464 else:
6465 raise Exception("Operation not supported")
6466 return result
6467
6468 - def GE(self, first, second):
6469 result = None
6470 name = self.search_fields[first.name]
6471 if name == "MESSAGE":
6472 last_message = self.get_last_message(first.tablename)
6473 result = "%s:%s" % (self.expand(second), last_message)
6474 elif name == "UID":
6475
6476
6477
6478 try:
6479 pedestal, threshold = self.get_uid_bounds(first.tablename)
6480 except TypeError:
6481 e = sys.exc_info()[1]
6482 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6483 return ""
6484 lower_limit = self.expand(second)
6485 result = "UID %s:%s" % (lower_limit, threshold)
6486 elif name == "DATE":
6487 result = "SINCE %s" % self.convert_date(second)
6488 else:
6489 raise Exception("Operation not supported")
6490 return result
6491
6492 - def LT(self, first, second):
6493 result = None
6494 name = self.search_fields[first.name]
6495 if name == "MESSAGE":
6496 result = "%s:%s" % (1, int(self.expand(second)) - 1)
6497 elif name == "UID":
6498 try:
6499 pedestal, threshold = self.get_uid_bounds(first.tablename)
6500 except TypeError:
6501 e = sys.exc_info()[1]
6502 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6503 return ""
6504 try:
6505 upper_limit = int(self.expand(second)) - 1
6506 except (ValueError, TypeError):
6507 e = sys.exc_info()[1]
6508 raise Exception("Operation not supported (non integer UID)")
6509 result = "UID %s:%s" % (pedestal, upper_limit)
6510 elif name == "DATE":
6511 result = "BEFORE %s" % self.convert_date(second)
6512 elif name == "SIZE":
6513 result = "SMALLER %s" % self.expand(second)
6514 else:
6515 raise Exception("Operation not supported")
6516 return result
6517
6518 - def LE(self, first, second):
6519 result = None
6520 name = self.search_fields[first.name]
6521 if name == "MESSAGE":
6522 result = "%s:%s" % (1, self.expand(second))
6523 elif name == "UID":
6524 try:
6525 pedestal, threshold = self.get_uid_bounds(first.tablename)
6526 except TypeError:
6527 e = sys.exc_info()[1]
6528 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6529 return ""
6530 upper_limit = int(self.expand(second))
6531 result = "UID %s:%s" % (pedestal, upper_limit)
6532 elif name == "DATE":
6533 result = "BEFORE %s" % self.convert_date(second, add=datetime.timedelta(1))
6534 else:
6535 raise Exception("Operation not supported")
6536 return result
6537
6538 - def NE(self, first, second=None):
6539 if (second is None) and isinstance(first, Field):
6540
6541 if first.type == "id":
6542 return self.GE(first, 1)
6543 result = self.NOT(self.EQ(first, second))
6544 result = result.replace("NOT NOT", "").strip()
6545 return result
6546
6547 - def EQ(self,first,second):
6548 name = self.search_fields[first.name]
6549 result = None
6550 if name is not None:
6551 if name == "MESSAGE":
6552
6553 result = "%s" % self.expand(second)
6554 elif name == "UID":
6555 result = "UID %s" % self.expand(second)
6556 elif name == "DATE":
6557 result = "ON %s" % self.convert_date(second)
6558
6559 elif name in self.flags:
6560 if second:
6561 result = "%s" % (name.upper()[1:])
6562 else:
6563 result = "NOT %s" % (name.upper()[1:])
6564 else:
6565 raise Exception("Operation not supported")
6566 else:
6567 raise Exception("Operation not supported")
6568 return result
6569
6570 - def AND(self, first, second):
6573
6574 - def OR(self, first, second):
6577
6578 - def NOT(self, first):
6579 result = "NOT %s" % self.expand(first)
6580 return result
6581
6582
6583
6584
6585
6586 ADAPTERS = {
6587 'sqlite': SQLiteAdapter,
6588 'spatialite': SpatiaLiteAdapter,
6589 'sqlite:memory': SQLiteAdapter,
6590 'spatialite:memory': SpatiaLiteAdapter,
6591 'mysql': MySQLAdapter,
6592 'postgres': PostgreSQLAdapter,
6593 'postgres:psycopg2': PostgreSQLAdapter,
6594 'postgres:pg8000': PostgreSQLAdapter,
6595 'postgres2:psycopg2': NewPostgreSQLAdapter,
6596 'postgres2:pg8000': NewPostgreSQLAdapter,
6597 'oracle': OracleAdapter,
6598 'mssql': MSSQLAdapter,
6599 'mssql2': MSSQL2Adapter,
6600 'mssql3': MSSQL3Adapter,
6601 'sybase': SybaseAdapter,
6602 'db2': DB2Adapter,
6603 'teradata': TeradataAdapter,
6604 'informix': InformixAdapter,
6605 'informix-se': InformixSEAdapter,
6606 'firebird': FireBirdAdapter,
6607 'firebird_embedded': FireBirdAdapter,
6608 'ingres': IngresAdapter,
6609 'ingresu': IngresUnicodeAdapter,
6610 'sapdb': SAPDBAdapter,
6611 'cubrid': CubridAdapter,
6612 'jdbc:sqlite': JDBCSQLiteAdapter,
6613 'jdbc:sqlite:memory': JDBCSQLiteAdapter,
6614 'jdbc:postgres': JDBCPostgreSQLAdapter,
6615 'gae': GoogleDatastoreAdapter,
6616 'google:datastore': GoogleDatastoreAdapter,
6617 'google:sql': GoogleSQLAdapter,
6618 'couchdb': CouchDBAdapter,
6619 'mongodb': MongoDBAdapter,
6620 'imap': IMAPAdapter
6621 }
6625 """
6626 Field type validation, using web2py's validators mechanism.
6627
6628 makes sure the content of a field is in line with the declared
6629 fieldtype
6630 """
6631 db = field.db
6632 if not have_validators:
6633 return []
6634 field_type, field_length = field.type, field.length
6635 if isinstance(field_type, SQLCustomType):
6636 if hasattr(field_type, 'validator'):
6637 return field_type.validator
6638 else:
6639 field_type = field_type.type
6640 elif not isinstance(field_type,str):
6641 return []
6642 requires=[]
6643 def ff(r,id):
6644 row=r(id)
6645 if not row:
6646 return id
6647 elif hasattr(r, '_format') and isinstance(r._format,str):
6648 return r._format % row
6649 elif hasattr(r, '_format') and callable(r._format):
6650 return r._format(row)
6651 else:
6652 return id
6653 if field_type in (('string', 'text', 'password')):
6654 requires.append(validators.IS_LENGTH(field_length))
6655 elif field_type == 'json':
6656 requires.append(validators.IS_EMPTY_OR(validators.IS_JSON()))
6657 elif field_type == 'double' or field_type == 'float':
6658 requires.append(validators.IS_FLOAT_IN_RANGE(-1e100, 1e100))
6659 elif field_type in ('integer','bigint'):
6660 requires.append(validators.IS_INT_IN_RANGE(-1e100, 1e100))
6661 elif field_type.startswith('decimal'):
6662 requires.append(validators.IS_DECIMAL_IN_RANGE(-10**10, 10**10))
6663 elif field_type == 'date':
6664 requires.append(validators.IS_DATE())
6665 elif field_type == 'time':
6666 requires.append(validators.IS_TIME())
6667 elif field_type == 'datetime':
6668 requires.append(validators.IS_DATETIME())
6669 elif db and field_type.startswith('reference') and \
6670 field_type.find('.') < 0 and \
6671 field_type[10:] in db.tables:
6672 referenced = db[field_type[10:]]
6673 def repr_ref(id, row=None, r=referenced, f=ff): return f(r, id)
6674 field.represent = field.represent or repr_ref
6675 if hasattr(referenced, '_format') and referenced._format:
6676 requires = validators.IS_IN_DB(db,referenced._id,
6677 referenced._format)
6678 if field.unique:
6679 requires._and = validators.IS_NOT_IN_DB(db,field)
6680 if field.tablename == field_type[10:]:
6681 return validators.IS_EMPTY_OR(requires)
6682 return requires
6683 elif db and field_type.startswith('list:reference') and \
6684 field_type.find('.') < 0 and \
6685 field_type[15:] in db.tables:
6686 referenced = db[field_type[15:]]
6687 def list_ref_repr(ids, row=None, r=referenced, f=ff):
6688 if not ids:
6689 return None
6690 refs = None
6691 db, id = r._db, r._id
6692 if isinstance(db._adapter, GoogleDatastoreAdapter):
6693 def count(values): return db(id.belongs(values)).select(id)
6694 rx = range(0, len(ids), 30)
6695 refs = reduce(lambda a,b:a&b, [count(ids[i:i+30]) for i in rx])
6696 else:
6697 refs = db(id.belongs(ids)).select(id)
6698 return (refs and ', '.join(str(f(r,x.id)) for x in refs) or '')
6699 field.represent = field.represent or list_ref_repr
6700 if hasattr(referenced, '_format') and referenced._format:
6701 requires = validators.IS_IN_DB(db,referenced._id,
6702 referenced._format,multiple=True)
6703 else:
6704 requires = validators.IS_IN_DB(db,referenced._id,
6705 multiple=True)
6706 if field.unique:
6707 requires._and = validators.IS_NOT_IN_DB(db,field)
6708 return requires
6709 elif field_type.startswith('list:'):
6710 def repr_list(values,row=None): return', '.join(str(v) for v in (values or []))
6711 field.represent = field.represent or repr_list
6712 if field.unique:
6713 requires.insert(0,validators.IS_NOT_IN_DB(db,field))
6714 sff = ['in', 'do', 'da', 'ti', 'de', 'bo']
6715 if field.notnull and not field_type[:2] in sff:
6716 requires.insert(0, validators.IS_NOT_EMPTY())
6717 elif not field.notnull and field_type[:2] in sff and requires:
6718 requires[-1] = validators.IS_EMPTY_OR(requires[-1])
6719 return requires
6720
6723 return str(item).replace('|', '||')
6724
6727
6729 if not hasattr(value,'split') and hasattr(value,'read'):
6730 value = value.read()
6731 return [int(x) for x in value.split('|') if x.strip()]
6732
6736
6737
6738 -class Row(object):
6739
6740 """
6741 a dictionary that lets you do d['a'] as well as d.a
6742 this is only used to store a Row
6743 """
6744
6746 self.__dict__.update(*args,**kwargs)
6747
6749 key=str(key)
6750 m = REGEX_TABLE_DOT_FIELD.match(key)
6751 if key in self.get('_extra',{}):
6752 return self._extra[key]
6753 elif m:
6754 try:
6755 return ogetattr(self, m.group(1))[m.group(2)]
6756 except (KeyError,AttributeError,TypeError):
6757 key = m.group(2)
6758 return ogetattr(self, key)
6759
6761 setattr(self, str(key), value)
6762
6763 __delitem__ = delattr
6764
6765 __copy__ = lambda self: Row(self)
6766
6767 __call__ = __getitem__
6768
6769 - def get(self,key,default=None):
6770 return self.__dict__.get(key,default)
6771
6773 return key in self.__dict__
6774
6775 has_key = __contains__
6776
6778 return len(self.__dict__)>0
6779
6780 - def update(self, *args, **kwargs):
6781 self.__dict__.update(*args, **kwargs)
6782
6784 return self.__dict__.keys()
6785
6787 return self.__dict__.items()
6788
6790 return self.__dict__.values()
6791
6794
6797
6799
6800 return '<Row %s>' % self.as_dict()
6801
6803 return '<Row %s>' % self.as_dict()
6804
6806 return object.__getattribute__(self,'id')
6807
6809 try:
6810 return self.as_dict() == other.as_dict()
6811 except AttributeError:
6812 return False
6813
6815 return not (self == other)
6816
6818 return Row(dict(self))
6819
6820 - def as_dict(self, datetime_to_str=False, custom_types=None):
6821 SERIALIZABLE_TYPES = [str, unicode, int, long, float, bool, list, dict]
6822 if isinstance(custom_types,(list,tuple,set)):
6823 SERIALIZABLE_TYPES += custom_types
6824 elif custom_types:
6825 SERIALIZABLE_TYPES.append(custom_types)
6826 d = dict(self)
6827 for k in copy.copy(d.keys()):
6828 v=d[k]
6829 if d[k] is None:
6830 continue
6831 elif isinstance(v,Row):
6832 d[k]=v.as_dict()
6833 elif isinstance(v,Reference):
6834 d[k]=int(v)
6835 elif isinstance(v,decimal.Decimal):
6836 d[k]=float(v)
6837 elif isinstance(v, (datetime.date, datetime.datetime, datetime.time)):
6838 if datetime_to_str:
6839 d[k] = v.isoformat().replace('T',' ')[:19]
6840 elif not isinstance(v,tuple(SERIALIZABLE_TYPES)):
6841 del d[k]
6842 return d
6843
6844 - def as_xml(self, row_name="row", colnames=None, indent=' '):
6845 def f(row,field,indent=' '):
6846 if isinstance(row,Row):
6847 spc = indent+' \n'
6848 items = [f(row[x],x,indent+' ') for x in row]
6849 return '%s<%s>\n%s\n%s</%s>' % (
6850 indent,
6851 field,
6852 spc.join(item for item in items if item),
6853 indent,
6854 field)
6855 elif not callable(row):
6856 if REGEX_ALPHANUMERIC.match(field):
6857 return '%s<%s>%s</%s>' % (indent,field,row,field)
6858 else:
6859 return '%s<extra name="%s">%s</extra>' % \
6860 (indent,field,row)
6861 else:
6862 return None
6863 return f(self, row_name, indent=indent)
6864
6865 - def as_json(self, mode="object", default=None, colnames=None,
6866 serialize=True, **kwargs):
6867 """
6868 serializes the table to a JSON list of objects
6869 kwargs are passed to .as_dict method
6870 only "object" mode supported for single row
6871
6872 serialize = False used by Rows.as_json
6873 TODO: return array mode with query column order
6874 """
6875
6876 def inner_loop(record, col):
6877 (t, f) = col.split('.')
6878 res = None
6879 if not REGEX_TABLE_DOT_FIELD.match(col):
6880 key = col
6881 res = record._extra[col]
6882 else:
6883 key = f
6884 if isinstance(record.get(t, None), Row):
6885 res = record[t][f]
6886 else:
6887 res = record[f]
6888 if mode == 'object':
6889 return (key, res)
6890 else:
6891 return res
6892
6893 multi = any([isinstance(v, self.__class__) for v in self.values()])
6894 mode = mode.lower()
6895 if not mode in ['object', 'array']:
6896 raise SyntaxError('Invalid JSON serialization mode: %s' % mode)
6897
6898 if mode=='object' and colnames:
6899 item = dict([inner_loop(self, col) for col in colnames])
6900 elif colnames:
6901 item = [inner_loop(self, col) for col in colnames]
6902 else:
6903 if not mode == 'object':
6904 raise SyntaxError('Invalid JSON serialization mode: %s' % mode)
6905
6906 if multi:
6907 item = dict()
6908 [item.update(**v.as_dict(**kwargs)) for v in self.values()]
6909 else:
6910 item = self.as_dict(**kwargs)
6911
6912 if serialize:
6913 if have_serializers:
6914 return serializers.json(item,
6915 default=default or
6916 serializers.custom_json)
6917 elif simplejson:
6918 return simplejson.dumps(item)
6919 else:
6920 raise RuntimeError("missing simplejson")
6921 else:
6922 return item
6923
6933
6935 if not isinstance(fields,(list,tuple)):
6936 fields = [fields]
6937 new_fields = []
6938 for field in fields:
6939 if isinstance(field,Field):
6940 new_fields.append(field)
6941 elif isinstance(field,Table):
6942 for ofield in field:
6943 new_fields.append(ofield)
6944 else:
6945 raise RuntimeError("fields must be a list of fields")
6946 fields = new_fields
6947 field_map = {}
6948 for field in fields:
6949 n = field.name.lower()
6950 if not n in field_map:
6951 field_map[n] = field
6952 n = str(field).lower()
6953 if not n in field_map:
6954 field_map[n] = field
6955 constants = {}
6956 i = 0
6957 while True:
6958 m = REGEX_CONST_STRING.search(text)
6959 if not m: break
6960 text = text[:m.start()]+('#%i' % i)+text[m.end():]
6961 constants[str(i)] = m.group()[1:-1]
6962 i+=1
6963 text = re.sub('\s+',' ',text).lower()
6964 for a,b in [('&','and'),
6965 ('|','or'),
6966 ('~','not'),
6967 ('==','='),
6968 ('<','<'),
6969 ('>','>'),
6970 ('<=','<='),
6971 ('>=','>='),
6972 ('<>','!='),
6973 ('=<','<='),
6974 ('=>','>='),
6975 ('=','='),
6976 (' less or equal than ','<='),
6977 (' greater or equal than ','>='),
6978 (' equal or less than ','<='),
6979 (' equal or greater than ','>='),
6980 (' less or equal ','<='),
6981 (' greater or equal ','>='),
6982 (' equal or less ','<='),
6983 (' equal or greater ','>='),
6984 (' not equal to ','!='),
6985 (' not equal ','!='),
6986 (' equal to ','='),
6987 (' equal ','='),
6988 (' equals ','='),
6989 (' less than ','<'),
6990 (' greater than ','>'),
6991 (' starts with ','startswith'),
6992 (' ends with ','endswith'),
6993 (' not in ' , 'notbelongs'),
6994 (' in ' , 'belongs'),
6995 (' is ','=')]:
6996 if a[0]==' ':
6997 text = text.replace(' is'+a,' %s ' % b)
6998 text = text.replace(a,' %s ' % b)
6999 text = re.sub('\s+',' ',text).lower()
7000 text = re.sub('(?P<a>[\<\>\!\=])\s+(?P<b>[\<\>\!\=])','\g<a>\g<b>',text)
7001 query = field = neg = op = logic = None
7002 for item in text.split():
7003 if field is None:
7004 if item == 'not':
7005 neg = True
7006 elif not neg and not logic and item in ('and','or'):
7007 logic = item
7008 elif item in field_map:
7009 field = field_map[item]
7010 else:
7011 raise RuntimeError("Invalid syntax")
7012 elif not field is None and op is None:
7013 op = item
7014 elif not op is None:
7015 if item.startswith('#'):
7016 if not item[1:] in constants:
7017 raise RuntimeError("Invalid syntax")
7018 value = constants[item[1:]]
7019 else:
7020 value = item
7021 if field.type in ('text', 'string', 'json'):
7022 if op == '=': op = 'like'
7023 if op == '=': new_query = field==value
7024 elif op == '<': new_query = field<value
7025 elif op == '>': new_query = field>value
7026 elif op == '<=': new_query = field<=value
7027 elif op == '>=': new_query = field>=value
7028 elif op == '!=': new_query = field!=value
7029 elif op == 'belongs': new_query = field.belongs(value.split(','))
7030 elif op == 'notbelongs': new_query = ~field.belongs(value.split(','))
7031 elif field.type in ('text', 'string', 'json'):
7032 if op == 'contains': new_query = field.contains(value)
7033 elif op == 'like': new_query = field.like(value)
7034 elif op == 'startswith': new_query = field.startswith(value)
7035 elif op == 'endswith': new_query = field.endswith(value)
7036 else: raise RuntimeError("Invalid operation")
7037 elif field._db._adapter.dbengine=='google:datastore' and \
7038 field.type in ('list:integer', 'list:string', 'list:reference'):
7039 if op == 'contains': new_query = field.contains(value)
7040 else: raise RuntimeError("Invalid operation")
7041 else: raise RuntimeError("Invalid operation")
7042 if neg: new_query = ~new_query
7043 if query is None:
7044 query = new_query
7045 elif logic == 'and':
7046 query &= new_query
7047 elif logic == 'or':
7048 query |= new_query
7049 field = op = neg = logic = None
7050 return query
7051
7053
7054 """
7055 an instance of this class represents a database connection
7056
7057 Example::
7058
7059 db = DAL('sqlite://test.db')
7060
7061 or
7062
7063 db = DAL({"uri": ..., "items": ...}) # experimental
7064
7065 db.define_table('tablename', Field('fieldname1'),
7066 Field('fieldname2'))
7067 """
7068
7069 - def __new__(cls, uri='sqlite://dummy.db', *args, **kwargs):
7096
7097 @staticmethod
7099 """
7100 # ## this allows gluon to set a folder for this thread
7101 # ## <<<<<<<<< Should go away as new DAL replaces old sql.py
7102 """
7103 BaseAdapter.set_folder(folder)
7104
7105 @staticmethod
7107 """
7108 Returns a dictionary with uri as key with timings and defined tables
7109 {'sqlite://storage.sqlite': {
7110 'dbstats': [(select auth_user.email from auth_user, 0.02009)],
7111 'dbtables': {
7112 'defined': ['auth_cas', 'auth_event', 'auth_group',
7113 'auth_membership', 'auth_permission', 'auth_user'],
7114 'lazy': '[]'
7115 }
7116 }
7117 }
7118 """
7119 dbs = getattr(THREAD_LOCAL,'db_instances',{}).items()
7120 infos = {}
7121 for db_uid, db_group in dbs:
7122 for db in db_group:
7123 if not db._uri:
7124 continue
7125 k = hide_password(db._uri)
7126 infos[k] = dict(dbstats = [(row[0], row[1]) for row in db._timings],
7127 dbtables = {'defined':
7128 sorted(list(set(db.tables) -
7129 set(db._LAZY_TABLES.keys()))),
7130 'lazy': sorted(db._LAZY_TABLES.keys())}
7131 )
7132 return infos
7133
7134 @staticmethod
7147
7148 @staticmethod
7170
7171 - def __init__(self, uri=DEFAULT_URI,
7172 pool_size=0, folder=None,
7173 db_codec='UTF-8', check_reserved=None,
7174 migrate=True, fake_migrate=False,
7175 migrate_enabled=True, fake_migrate_all=False,
7176 decode_credentials=False, driver_args=None,
7177 adapter_args=None, attempts=5, auto_import=False,
7178 bigint_id=False,debug=False,lazy_tables=False,
7179 db_uid=None, do_connect=True, after_connection=None):
7180 """
7181 Creates a new Database Abstraction Layer instance.
7182
7183 Keyword arguments:
7184
7185 :uri: string that contains information for connecting to a database.
7186 (default: 'sqlite://dummy.db')
7187
7188 experimental: you can specify a dictionary as uri
7189 parameter i.e. with
7190 db = DAL({"uri": "sqlite://storage.sqlite",
7191 "items": {...}, ...})
7192
7193 for an example of dict input you can check the output
7194 of the scaffolding db model with
7195
7196 db.as_dict()
7197
7198 Note that for compatibility with Python older than
7199 version 2.6.5 you should cast your dict input keys
7200 to str due to a syntax limitation on kwarg names.
7201 for proper DAL dictionary input you can use one of:
7202
7203 obj = serializers.cast_keys(dict, [encoding="utf-8"])
7204
7205 or else (for parsing json input)
7206
7207 obj = serializers.loads_json(data, unicode_keys=False)
7208
7209 :pool_size: How many open connections to make to the database object.
7210 :folder: where .table files will be created.
7211 automatically set within web2py
7212 use an explicit path when using DAL outside web2py
7213 :db_codec: string encoding of the database (default: 'UTF-8')
7214 :check_reserved: list of adapters to check tablenames and column names
7215 against sql/nosql reserved keywords. (Default None)
7216
7217 * 'common' List of sql keywords that are common to all database types
7218 such as "SELECT, INSERT". (recommended)
7219 * 'all' Checks against all known SQL keywords. (not recommended)
7220 <adaptername> Checks against the specific adapters list of keywords
7221 (recommended)
7222 * '<adaptername>_nonreserved' Checks against the specific adapters
7223 list of nonreserved keywords. (if available)
7224 :migrate (defaults to True) sets default migrate behavior for all tables
7225 :fake_migrate (defaults to False) sets default fake_migrate behavior for all tables
7226 :migrate_enabled (defaults to True). If set to False disables ALL migrations
7227 :fake_migrate_all (defaults to False). If sets to True fake migrates ALL tables
7228 :attempts (defaults to 5). Number of times to attempt connecting
7229 :auto_import (defaults to False). If set, import automatically table definitions from the
7230 databases folder
7231 :bigint_id (defaults to False): If set, turn on bigint instead of int for id fields
7232 :lazy_tables (defaults to False): delay table definition until table access
7233 :after_connection (defaults to None): a callable that will be execute after the connection
7234 """
7235
7236 items = None
7237 if isinstance(uri, dict):
7238 if "items" in uri:
7239 items = uri.pop("items")
7240 try:
7241 newuri = uri.pop("uri")
7242 except KeyError:
7243 newuri = DEFAULT_URI
7244 locals().update(uri)
7245 uri = newuri
7246
7247 if uri == '<zombie>' and db_uid is not None: return
7248 if not decode_credentials:
7249 credential_decoder = lambda cred: cred
7250 else:
7251 credential_decoder = lambda cred: urllib.unquote(cred)
7252 self._folder = folder
7253 if folder:
7254 self.set_folder(folder)
7255 self._uri = uri
7256 self._pool_size = pool_size
7257 self._db_codec = db_codec
7258 self._lastsql = ''
7259 self._timings = []
7260 self._pending_references = {}
7261 self._request_tenant = 'request_tenant'
7262 self._common_fields = []
7263 self._referee_name = '%(table)s'
7264 self._bigint_id = bigint_id
7265 self._debug = debug
7266 self._migrated = []
7267 self._LAZY_TABLES = {}
7268 self._lazy_tables = lazy_tables
7269 self._tables = SQLCallableList()
7270 self._driver_args = driver_args
7271 self._adapter_args = adapter_args
7272 self._check_reserved = check_reserved
7273 self._decode_credentials = decode_credentials
7274 self._attempts = attempts
7275 self._do_connect = do_connect
7276
7277 if not str(attempts).isdigit() or attempts < 0:
7278 attempts = 5
7279 if uri:
7280 uris = isinstance(uri,(list,tuple)) and uri or [uri]
7281 error = ''
7282 connected = False
7283 for k in range(attempts):
7284 for uri in uris:
7285 try:
7286 if is_jdbc and not uri.startswith('jdbc:'):
7287 uri = 'jdbc:'+uri
7288 self._dbname = REGEX_DBNAME.match(uri).group()
7289 if not self._dbname in ADAPTERS:
7290 raise SyntaxError("Error in URI '%s' or database not supported" % self._dbname)
7291
7292
7293 kwargs = dict(db=self,uri=uri,
7294 pool_size=pool_size,
7295 folder=folder,
7296 db_codec=db_codec,
7297 credential_decoder=credential_decoder,
7298 driver_args=driver_args or {},
7299 adapter_args=adapter_args or {},
7300 do_connect=do_connect,
7301 after_connection=after_connection)
7302 self._adapter = ADAPTERS[self._dbname](**kwargs)
7303 types = ADAPTERS[self._dbname].types
7304
7305 self._adapter.types = copy.copy(types)
7306 if bigint_id:
7307 if 'big-id' in types and 'reference' in types:
7308 self._adapter.types['id'] = types['big-id']
7309 self._adapter.types['reference'] = types['big-reference']
7310 connected = True
7311 break
7312 except SyntaxError:
7313 raise
7314 except Exception:
7315 tb = traceback.format_exc()
7316 sys.stderr.write('DEBUG: connect attempt %i, connection error:\n%s' % (k, tb))
7317 if connected:
7318 break
7319 else:
7320 time.sleep(1)
7321 if not connected:
7322 raise RuntimeError("Failure to connect, tried %d times:\n%s" % (attempts, tb))
7323 else:
7324 self._adapter = BaseAdapter(db=self,pool_size=0,
7325 uri='None',folder=folder,
7326 db_codec=db_codec, after_connection=after_connection)
7327 migrate = fake_migrate = False
7328 adapter = self._adapter
7329 self._uri_hash = hashlib_md5(adapter.uri).hexdigest()
7330 self.check_reserved = check_reserved
7331 if self.check_reserved:
7332 from reserved_sql_keywords import ADAPTERS as RSK
7333 self.RSK = RSK
7334 self._migrate = migrate
7335 self._fake_migrate = fake_migrate
7336 self._migrate_enabled = migrate_enabled
7337 self._fake_migrate_all = fake_migrate_all
7338 if auto_import or items:
7339 self.import_table_definitions(adapter.folder,
7340 items=items)
7341
7342 @property
7345
7348 pattern = pjoin(path,self._uri_hash+'_*.table')
7349 if items:
7350 for tablename, table in items.iteritems():
7351
7352 fields = []
7353
7354 [table.pop(name) for name in ("name", "fields") if
7355 name in table]
7356 if "items" in table:
7357 for fieldname, field in table.pop("items").iteritems():
7358
7359 [field.pop(key) for key in ("requires", "name",
7360 "compute", "colname") if key in field]
7361 fields.append(Field(str(fieldname), **field))
7362 self.define_table(str(tablename), *fields, **table)
7363 else:
7364 for filename in glob.glob(pattern):
7365 tfile = self._adapter.file_open(filename, 'r')
7366 try:
7367 sql_fields = pickle.load(tfile)
7368 name = filename[len(pattern)-7:-6]
7369 mf = [(value['sortable'],
7370 Field(key,
7371 type=value['type'],
7372 length=value.get('length',None),
7373 notnull=value.get('notnull',False),
7374 unique=value.get('unique',False))) \
7375 for key, value in sql_fields.iteritems()]
7376 mf.sort(lambda a,b: cmp(a[0],b[0]))
7377 self.define_table(name,*[item[1] for item in mf],
7378 **dict(migrate=migrate,
7379 fake_migrate=fake_migrate))
7380 finally:
7381 self._adapter.file_close(tfile)
7382
7384 """
7385 Validates ``name`` against SQL keywords
7386 Uses self.check_reserve which is a list of
7387 operators to use.
7388 self.check_reserved
7389 ['common', 'postgres', 'mysql']
7390 self.check_reserved
7391 ['all']
7392 """
7393 for backend in self.check_reserved:
7394 if name.upper() in self.RSK[backend]:
7395 raise SyntaxError(
7396 'invalid table/column name "%s" is a "%s" reserved SQL/NOSQL keyword' % (name, backend.upper()))
7397
7398 - def parse_as_rest(self,patterns,args,vars,queries=None,nested_select=True):
7399 """
7400 EXAMPLE:
7401
7402 db.define_table('person',Field('name'),Field('info'))
7403 db.define_table('pet',Field('ownedby',db.person),Field('name'),Field('info'))
7404
7405 @request.restful()
7406 def index():
7407 def GET(*args,**vars):
7408 patterns = [
7409 "/friends[person]",
7410 "/{person.name}/:field",
7411 "/{person.name}/pets[pet.ownedby]",
7412 "/{person.name}/pets[pet.ownedby]/{pet.name}",
7413 "/{person.name}/pets[pet.ownedby]/{pet.name}/:field",
7414 ("/dogs[pet]", db.pet.info=='dog'),
7415 ("/dogs[pet]/{pet.name.startswith}", db.pet.info=='dog'),
7416 ]
7417 parser = db.parse_as_rest(patterns,args,vars)
7418 if parser.status == 200:
7419 return dict(content=parser.response)
7420 else:
7421 raise HTTP(parser.status,parser.error)
7422
7423 def POST(table_name,**vars):
7424 if table_name == 'person':
7425 return db.person.validate_and_insert(**vars)
7426 elif table_name == 'pet':
7427 return db.pet.validate_and_insert(**vars)
7428 else:
7429 raise HTTP(400)
7430 return locals()
7431 """
7432
7433 db = self
7434 re1 = REGEX_SEARCH_PATTERN
7435 re2 = REGEX_SQUARE_BRACKETS
7436
7437 def auto_table(table,base='',depth=0):
7438 patterns = []
7439 for field in db[table].fields:
7440 if base:
7441 tag = '%s/%s' % (base,field.replace('_','-'))
7442 else:
7443 tag = '/%s/%s' % (table.replace('_','-'),field.replace('_','-'))
7444 f = db[table][field]
7445 if not f.readable: continue
7446 if f.type=='id' or 'slug' in field or f.type.startswith('reference'):
7447 tag += '/{%s.%s}' % (table,field)
7448 patterns.append(tag)
7449 patterns.append(tag+'/:field')
7450 elif f.type.startswith('boolean'):
7451 tag += '/{%s.%s}' % (table,field)
7452 patterns.append(tag)
7453 patterns.append(tag+'/:field')
7454 elif f.type in ('float','double','integer','bigint'):
7455 tag += '/{%s.%s.ge}/{%s.%s.lt}' % (table,field,table,field)
7456 patterns.append(tag)
7457 patterns.append(tag+'/:field')
7458 elif f.type.startswith('list:'):
7459 tag += '/{%s.%s.contains}' % (table,field)
7460 patterns.append(tag)
7461 patterns.append(tag+'/:field')
7462 elif f.type in ('date','datetime'):
7463 tag+= '/{%s.%s.year}' % (table,field)
7464 patterns.append(tag)
7465 patterns.append(tag+'/:field')
7466 tag+='/{%s.%s.month}' % (table,field)
7467 patterns.append(tag)
7468 patterns.append(tag+'/:field')
7469 tag+='/{%s.%s.day}' % (table,field)
7470 patterns.append(tag)
7471 patterns.append(tag+'/:field')
7472 if f.type in ('datetime','time'):
7473 tag+= '/{%s.%s.hour}' % (table,field)
7474 patterns.append(tag)
7475 patterns.append(tag+'/:field')
7476 tag+='/{%s.%s.minute}' % (table,field)
7477 patterns.append(tag)
7478 patterns.append(tag+'/:field')
7479 tag+='/{%s.%s.second}' % (table,field)
7480 patterns.append(tag)
7481 patterns.append(tag+'/:field')
7482 if depth>0:
7483 for f in db[table]._referenced_by:
7484 tag+='/%s[%s.%s]' % (table,f.tablename,f.name)
7485 patterns.append(tag)
7486 patterns += auto_table(table,base=tag,depth=depth-1)
7487 return patterns
7488
7489 if patterns == 'auto':
7490 patterns=[]
7491 for table in db.tables:
7492 if not table.startswith('auth_'):
7493 patterns.append('/%s[%s]' % (table,table))
7494 patterns += auto_table(table,base='',depth=1)
7495 else:
7496 i = 0
7497 while i<len(patterns):
7498 pattern = patterns[i]
7499 if not isinstance(pattern,str):
7500 pattern = pattern[0]
7501 tokens = pattern.split('/')
7502 if tokens[-1].startswith(':auto') and re2.match(tokens[-1]):
7503 new_patterns = auto_table(tokens[-1][tokens[-1].find('[')+1:-1],
7504 '/'.join(tokens[:-1]))
7505 patterns = patterns[:i]+new_patterns+patterns[i+1:]
7506 i += len(new_patterns)
7507 else:
7508 i += 1
7509 if '/'.join(args) == 'patterns':
7510 return Row({'status':200,'pattern':'list',
7511 'error':None,'response':patterns})
7512 for pattern in patterns:
7513 basequery, exposedfields = None, []
7514 if isinstance(pattern,tuple):
7515 if len(pattern)==2:
7516 pattern, basequery = pattern
7517 elif len(pattern)>2:
7518 pattern, basequery, exposedfields = pattern[0:3]
7519 otable=table=None
7520 if not isinstance(queries,dict):
7521 dbset=db(queries)
7522 if basequery is not None:
7523 dbset = dbset(basequery)
7524 i=0
7525 tags = pattern[1:].split('/')
7526 if len(tags)!=len(args):
7527 continue
7528 for tag in tags:
7529 if re1.match(tag):
7530
7531 tokens = tag[1:-1].split('.')
7532 table, field = tokens[0], tokens[1]
7533 if not otable or table == otable:
7534 if len(tokens)==2 or tokens[2]=='eq':
7535 query = db[table][field]==args[i]
7536 elif tokens[2]=='ne':
7537 query = db[table][field]!=args[i]
7538 elif tokens[2]=='lt':
7539 query = db[table][field]<args[i]
7540 elif tokens[2]=='gt':
7541 query = db[table][field]>args[i]
7542 elif tokens[2]=='ge':
7543 query = db[table][field]>=args[i]
7544 elif tokens[2]=='le':
7545 query = db[table][field]<=args[i]
7546 elif tokens[2]=='year':
7547 query = db[table][field].year()==args[i]
7548 elif tokens[2]=='month':
7549 query = db[table][field].month()==args[i]
7550 elif tokens[2]=='day':
7551 query = db[table][field].day()==args[i]
7552 elif tokens[2]=='hour':
7553 query = db[table][field].hour()==args[i]
7554 elif tokens[2]=='minute':
7555 query = db[table][field].minutes()==args[i]
7556 elif tokens[2]=='second':
7557 query = db[table][field].seconds()==args[i]
7558 elif tokens[2]=='startswith':
7559 query = db[table][field].startswith(args[i])
7560 elif tokens[2]=='contains':
7561 query = db[table][field].contains(args[i])
7562 else:
7563 raise RuntimeError("invalid pattern: %s" % pattern)
7564 if len(tokens)==4 and tokens[3]=='not':
7565 query = ~query
7566 elif len(tokens)>=4:
7567 raise RuntimeError("invalid pattern: %s" % pattern)
7568 if not otable and isinstance(queries,dict):
7569 dbset = db(queries[table])
7570 if basequery is not None:
7571 dbset = dbset(basequery)
7572 dbset=dbset(query)
7573 else:
7574 raise RuntimeError("missing relation in pattern: %s" % pattern)
7575 elif re2.match(tag) and args[i]==tag[:tag.find('[')]:
7576 ref = tag[tag.find('[')+1:-1]
7577 if '.' in ref and otable:
7578 table,field = ref.split('.')
7579 selfld = '_id'
7580 if db[table][field].type.startswith('reference '):
7581 refs = [ x.name for x in db[otable] if x.type == db[table][field].type ]
7582 else:
7583 refs = [ x.name for x in db[table]._referenced_by if x.tablename==otable ]
7584 if refs:
7585 selfld = refs[0]
7586 if nested_select:
7587 try:
7588 dbset=db(db[table][field].belongs(dbset._select(db[otable][selfld])))
7589 except ValueError:
7590 return Row({'status':400,'pattern':pattern,
7591 'error':'invalid path','response':None})
7592 else:
7593 items = [item.id for item in dbset.select(db[otable][selfld])]
7594 dbset=db(db[table][field].belongs(items))
7595 else:
7596 table = ref
7597 if not otable and isinstance(queries,dict):
7598 dbset = db(queries[table])
7599 dbset=dbset(db[table])
7600 elif tag==':field' and table:
7601
7602 field = args[i]
7603 if not field in db[table]: break
7604
7605 if not db[table][field].readable:
7606 return Row({'status':418,'pattern':pattern,
7607 'error':'I\'m a teapot','response':None})
7608 try:
7609 distinct = vars.get('distinct', False) == 'True'
7610 offset = int(vars.get('offset',None) or 0)
7611 limits = (offset,int(vars.get('limit',None) or 1000)+offset)
7612 except ValueError:
7613 return Row({'status':400,'error':'invalid limits','response':None})
7614 items = dbset.select(db[table][field], distinct=distinct, limitby=limits)
7615 if items:
7616 return Row({'status':200,'response':items,
7617 'pattern':pattern})
7618 else:
7619 return Row({'status':404,'pattern':pattern,
7620 'error':'no record found','response':None})
7621 elif tag != args[i]:
7622 break
7623 otable = table
7624 i += 1
7625 if i==len(tags) and table:
7626 ofields = vars.get('order',db[table]._id.name).split('|')
7627 try:
7628 orderby = [db[table][f] if not f.startswith('~') else ~db[table][f[1:]] for f in ofields]
7629 except (KeyError, AttributeError):
7630 return Row({'status':400,'error':'invalid orderby','response':None})
7631 if exposedfields:
7632 fields = [field for field in db[table] if str(field).split('.')[-1] in exposedfields and field.readable]
7633 else:
7634 fields = [field for field in db[table] if field.readable]
7635 count = dbset.count()
7636 try:
7637 offset = int(vars.get('offset',None) or 0)
7638 limits = (offset,int(vars.get('limit',None) or 1000)+offset)
7639 except ValueError:
7640 return Row({'status':400,'error':'invalid limits','response':None})
7641 if count > limits[1]-limits[0]:
7642 return Row({'status':400,'error':'too many records','response':None})
7643 try:
7644 response = dbset.select(limitby=limits,orderby=orderby,*fields)
7645 except ValueError:
7646 return Row({'status':400,'pattern':pattern,
7647 'error':'invalid path','response':None})
7648 return Row({'status':200,'response':response,
7649 'pattern':pattern,'count':count})
7650 return Row({'status':400,'error':'no matching pattern','response':None})
7651
7652 - def define_table(
7653 self,
7654 tablename,
7655 *fields,
7656 **args
7657 ):
7658 if not isinstance(tablename,str):
7659 raise SyntaxError("missing table name")
7660 elif hasattr(self,tablename) or tablename in self.tables:
7661 if not args.get('redefine',False):
7662 raise SyntaxError('table already defined: %s' % tablename)
7663 elif tablename.startswith('_') or hasattr(self,tablename) or \
7664 REGEX_PYTHON_KEYWORDS.match(tablename):
7665 raise SyntaxError('invalid table name: %s' % tablename)
7666 elif self.check_reserved:
7667 self.check_reserved_keyword(tablename)
7668 else:
7669 invalid_args = set(args)-TABLE_ARGS
7670 if invalid_args:
7671 raise SyntaxError('invalid table "%s" attributes: %s' \
7672 % (tablename,invalid_args))
7673 if self._lazy_tables and not tablename in self._LAZY_TABLES:
7674 self._LAZY_TABLES[tablename] = (tablename,fields,args)
7675 table = None
7676 else:
7677 table = self.lazy_define_table(tablename,*fields,**args)
7678 if not tablename in self.tables:
7679 self.tables.append(tablename)
7680 return table
7681
7682 - def lazy_define_table(
7683 self,
7684 tablename,
7685 *fields,
7686 **args
7687 ):
7688 args_get = args.get
7689 common_fields = self._common_fields
7690 if common_fields:
7691 fields = list(fields) + list(common_fields)
7692
7693 table_class = args_get('table_class',Table)
7694 table = table_class(self, tablename, *fields, **args)
7695 table._actual = True
7696 self[tablename] = table
7697
7698 table._create_references()
7699 for field in table:
7700 if field.requires == DEFAULT:
7701 field.requires = sqlhtml_validators(field)
7702
7703 migrate = self._migrate_enabled and args_get('migrate',self._migrate)
7704 if migrate and not self._uri in (None,'None') \
7705 or self._adapter.dbengine=='google:datastore':
7706 fake_migrate = self._fake_migrate_all or \
7707 args_get('fake_migrate',self._fake_migrate)
7708 polymodel = args_get('polymodel',None)
7709 try:
7710 GLOBAL_LOCKER.acquire()
7711 self._lastsql = self._adapter.create_table(
7712 table,migrate=migrate,
7713 fake_migrate=fake_migrate,
7714 polymodel=polymodel)
7715 finally:
7716 GLOBAL_LOCKER.release()
7717 else:
7718 table._dbt = None
7719 on_define = args_get('on_define',None)
7720 if on_define: on_define(table)
7721 return table
7722
7723 - def as_dict(self, flat=False, sanitize=True, field_options=True):
7724 dbname = db_uid = uri = None
7725 if not sanitize:
7726 uri, dbname, db_uid = (self._uri, self._dbname, self._db_uid)
7727 db_as_dict = dict(items={}, tables=[], uri=uri, dbname=dbname,
7728 db_uid=db_uid,
7729 **dict([(k, getattr(self, "_" + k)) for
7730 k in 'pool_size','folder','db_codec',
7731 'check_reserved','migrate','fake_migrate',
7732 'migrate_enabled','fake_migrate_all',
7733 'decode_credentials','driver_args',
7734 'adapter_args', 'attempts',
7735 'bigint_id','debug','lazy_tables',
7736 'do_connect']))
7737
7738 for table in self:
7739 tablename = str(table)
7740 db_as_dict["tables"].append(tablename)
7741 db_as_dict["items"][tablename] = table.as_dict(flat=flat,
7742 sanitize=sanitize,
7743 field_options=field_options)
7744 return db_as_dict
7745
7746 - def as_xml(self, sanitize=True, field_options=True):
7752
7753 - def as_json(self, sanitize=True, field_options=True):
7759
7760 - def as_yaml(self, sanitize=True, field_options=True):
7766
7768 try:
7769 return tablename in self.tables
7770 except AttributeError:
7771
7772 return False
7773
7774 has_key = __contains__
7775
7776 - def get(self,key,default=None):
7777 return self.__dict__.get(key,default)
7778
7780 for tablename in self.tables:
7781 yield self[tablename]
7782
7785
7787 if ogetattr(self,'_lazy_tables') and \
7788 key in ogetattr(self,'_LAZY_TABLES'):
7789 tablename, fields, args = self._LAZY_TABLES.pop(key)
7790 return self.lazy_define_table(tablename,*fields,**args)
7791 return ogetattr(self, key)
7792
7794 osetattr(self, str(key), value)
7795
7797 if key[:1]!='_' and key in self:
7798 raise SyntaxError(
7799 'Object %s exists and cannot be redefined' % key)
7800 osetattr(self,key,value)
7801
7802 __delitem__ = object.__delattr__
7803
7805 if hasattr(self,'_uri'):
7806 return '<DAL uri="%s">' % hide_password(str(self._uri))
7807 else:
7808 return '<DAL db_uid="%s">' % self._db_uid
7809
7812
7813 - def __call__(self, query=None, ignore_common_filters=None):
7814 if isinstance(query,Table):
7815 query = self._adapter.id_query(query)
7816 elif isinstance(query,Field):
7817 query = query!=None
7818 elif isinstance(query, dict):
7819 icf = query.get("ignore_common_filters")
7820 if icf: ignore_common_filters = icf
7821 return Set(self, query, ignore_common_filters=ignore_common_filters)
7822
7825
7828
7830 self._adapter.close()
7831 if self._db_uid in THREAD_LOCAL.db_instances:
7832 db_group = THREAD_LOCAL.db_instances[self._db_uid]
7833 db_group.remove(self)
7834 if not db_group:
7835 del THREAD_LOCAL.db_instances[self._db_uid]
7836
7837 - def executesql(self, query, placeholders=None, as_dict=False,
7838 fields=None, colnames=None):
7839 """
7840 placeholders is optional and will always be None.
7841 If using raw SQL with placeholders, placeholders may be
7842 a sequence of values to be substituted in
7843 or, (if supported by the DB driver), a dictionary with keys
7844 matching named placeholders in your SQL.
7845
7846 Added 2009-12-05 "as_dict" optional argument. Will always be
7847 None when using DAL. If using raw SQL can be set to True
7848 and the results cursor returned by the DB driver will be
7849 converted to a sequence of dictionaries keyed with the db
7850 field names. Tested with SQLite but should work with any database
7851 since the cursor.description used to get field names is part of the
7852 Python dbi 2.0 specs. Results returned with as_dict=True are
7853 the same as those returned when applying .to_list() to a DAL query.
7854
7855 [{field1: value1, field2: value2}, {field1: value1b, field2: value2b}]
7856
7857 Added 2012-08-24 "fields" and "colnames" optional arguments. If either
7858 is provided, the results cursor returned by the DB driver will be
7859 converted to a DAL Rows object using the db._adapter.parse() method.
7860
7861 The "fields" argument is a list of DAL Field objects that match the
7862 fields returned from the DB. The Field objects should be part of one or
7863 more Table objects defined on the DAL object. The "fields" list can
7864 include one or more DAL Table objects in addition to or instead of
7865 including Field objects, or it can be just a single table (not in a
7866 list). In that case, the Field objects will be extracted from the
7867 table(s).
7868
7869 Instead of specifying the "fields" argument, the "colnames" argument
7870 can be specified as a list of field names in tablename.fieldname format.
7871 Again, these should represent tables and fields defined on the DAL
7872 object.
7873
7874 It is also possible to specify both "fields" and the associated
7875 "colnames". In that case, "fields" can also include DAL Expression
7876 objects in addition to Field objects. For Field objects in "fields",
7877 the associated "colnames" must still be in tablename.fieldname format.
7878 For Expression objects in "fields", the associated "colnames" can
7879 be any arbitrary labels.
7880
7881 Note, the DAL Table objects referred to by "fields" or "colnames" can
7882 be dummy tables and do not have to represent any real tables in the
7883 database. Also, note that the "fields" and "colnames" must be in the
7884 same order as the fields in the results cursor returned from the DB.
7885 """
7886 adapter = self._adapter
7887 if placeholders:
7888 adapter.execute(query, placeholders)
7889 else:
7890 adapter.execute(query)
7891 if as_dict:
7892 if not hasattr(adapter.cursor,'description'):
7893 raise RuntimeError("database does not support executesql(...,as_dict=True)")
7894
7895
7896
7897 columns = adapter.cursor.description
7898
7899 fields = [f[0] for f in columns]
7900
7901 data = adapter._fetchall()
7902
7903
7904 return [dict(zip(fields,row)) for row in data]
7905 try:
7906 data = adapter._fetchall()
7907 except:
7908 return None
7909 if fields or colnames:
7910 fields = [] if fields is None else fields
7911 if not isinstance(fields, list):
7912 fields = [fields]
7913 extracted_fields = []
7914 for field in fields:
7915 if isinstance(field, Table):
7916 extracted_fields.extend([f for f in field])
7917 else:
7918 extracted_fields.append(field)
7919 if not colnames:
7920 colnames = ['%s.%s' % (f.tablename, f.name)
7921 for f in extracted_fields]
7922 data = adapter.parse(
7923 data, fields=extracted_fields, colnames=colnames)
7924 return data
7925
7927 for table in self:
7928 table._referenced_by = [field for field in table._referenced_by
7929 if not field.table==thistable]
7930
7932 step = int(kwargs.get('max_fetch_rows,',500))
7933 write_colnames = kwargs['write_colnames'] = \
7934 kwargs.get("write_colnames", True)
7935 for table in self.tables:
7936 ofile.write('TABLE %s\r\n' % table)
7937 query = self._adapter.id_query(self[table])
7938 nrows = self(query).count()
7939 kwargs['write_colnames'] = write_colnames
7940 for k in range(0,nrows,step):
7941 self(query).select(limitby=(k,k+step)).export_to_csv_file(
7942 ofile, *args, **kwargs)
7943 kwargs['write_colnames'] = False
7944 ofile.write('\r\n\r\n')
7945 ofile.write('END')
7946
7947 - def import_from_csv_file(self, ifile, id_map=None, null='<NULL>',
7948 unique='uuid', *args, **kwargs):
7949
7950 id_offset = {}
7951 for line in ifile:
7952 line = line.strip()
7953 if not line:
7954 continue
7955 elif line == 'END':
7956 return
7957 elif not line.startswith('TABLE ') or not line[6:] in self.tables:
7958 raise SyntaxError('invalid file format')
7959 else:
7960 tablename = line[6:]
7961 self[tablename].import_from_csv_file(
7962 ifile, id_map, null, unique, id_offset, *args, **kwargs)
7963
7965 return DAL('<zombie>',db_uid=db_uid)
7966
7969
7970 copyreg.pickle(DAL, DAL_pickler, DAL_unpickler)
7973 """
7974 Helper class providing a comma-separated string having all the field names
7975 (prefixed by table name and '.')
7976
7977 normally only called from within gluon.sql
7978 """
7979
7982
7984 return ', '.join([str(field) for field in self._table])
7985
7988
7990 if not self._record:
7991 self._record = self._table[int(self)]
7992 if not self._record:
7993 raise RuntimeError(
7994 "Using a recursive select but encountered a broken reference: %s %d"%(self._table, int(self)))
7995
7997 if key == 'id':
7998 return int(self)
7999 self.__allocate()
8000 return self._record.get(key, None)
8001
8002 - def get(self, key, default=None):
8004
8011
8013 if key == 'id':
8014 return int(self)
8015 self.__allocate()
8016 return self._record.get(key, None)
8017
8019 self.__allocate()
8020 self._record[key] = value
8021
8024 return marshal.loads(data)
8025
8027 try:
8028 marshal_dump = marshal.dumps(int(data))
8029 except AttributeError:
8030 marshal_dump = 'i%s' % struct.pack('<i', int(data))
8031 return (Reference_unpickler, (marshal_dump,))
8032
8033 copyreg.pickle(Reference, Reference_pickler, Reference_unpickler)
8034
8035
8036 -class Table(object):
8037
8038 """
8039 an instance of this class represents a database table
8040
8041 Example::
8042
8043 db = DAL(...)
8044 db.define_table('users', Field('name'))
8045 db.users.insert(name='me') # print db.users._insert(...) to see SQL
8046 db.users.drop()
8047 """
8048
8049 - def __init__(
8050 self,
8051 db,
8052 tablename,
8053 *fields,
8054 **args
8055 ):
8056 """
8057 Initializes the table and performs checking on the provided fields.
8058
8059 Each table will have automatically an 'id'.
8060
8061 If a field is of type Table, the fields (excluding 'id') from that table
8062 will be used instead.
8063
8064 :raises SyntaxError: when a supplied field is of incorrect type.
8065 """
8066 self._actual = False
8067 self._tablename = tablename
8068 self._sequence_name = args.get('sequence_name',None) or \
8069 db and db._adapter.sequence_name(tablename)
8070 self._trigger_name = args.get('trigger_name',None) or \
8071 db and db._adapter.trigger_name(tablename)
8072 self._common_filter = args.get('common_filter', None)
8073 self._format = args.get('format',None)
8074 self._singular = args.get(
8075 'singular',tablename.replace('_',' ').capitalize())
8076 self._plural = args.get(
8077 'plural',pluralize(self._singular.lower()).capitalize())
8078
8079 if 'primarykey' in args and args['primarykey']:
8080 self._primarykey = args.get('primarykey', None)
8081
8082 self._before_insert = []
8083 self._before_update = [Set.delete_uploaded_files]
8084 self._before_delete = [Set.delete_uploaded_files]
8085 self._after_insert = []
8086 self._after_update = []
8087 self._after_delete = []
8088
8089 fieldnames,newfields=set(),[]
8090 if hasattr(self,'_primarykey'):
8091 if not isinstance(self._primarykey,list):
8092 raise SyntaxError(
8093 "primarykey must be a list of fields from table '%s'" \
8094 % tablename)
8095 if len(self._primarykey)==1:
8096 self._id = [f for f in fields if isinstance(f,Field) \
8097 and f.name==self._primarykey[0]][0]
8098 elif not [f for f in fields if isinstance(f,Field) and f.type=='id']:
8099 field = Field('id', 'id')
8100 newfields.append(field)
8101 fieldnames.add('id')
8102 self._id = field
8103 virtual_fields = []
8104 for field in fields:
8105 if isinstance(field, (FieldMethod, FieldVirtual)):
8106 virtual_fields.append(field)
8107 elif isinstance(field, Field) and not field.name in fieldnames:
8108 if field.db is not None:
8109 field = copy.copy(field)
8110 newfields.append(field)
8111 fieldnames.add(field.name)
8112 if field.type=='id':
8113 self._id = field
8114 elif isinstance(field, Table):
8115 table = field
8116 for field in table:
8117 if not field.name in fieldnames and not field.type=='id':
8118 t2 = not table._actual and self._tablename
8119 field = field.clone(point_self_references_to=t2)
8120 newfields.append(field)
8121 fieldnames.add(field.name)
8122 elif not isinstance(field, (Field, Table)):
8123 raise SyntaxError(
8124 'define_table argument is not a Field or Table: %s' % field)
8125 fields = newfields
8126 self._db = db
8127 tablename = tablename
8128 self._fields = SQLCallableList()
8129 self.virtualfields = []
8130 fields = list(fields)
8131
8132 if db and db._adapter.uploads_in_blob==True:
8133 uploadfields = [f.name for f in fields if f.type=='blob']
8134 for field in fields:
8135 fn = field.uploadfield
8136 if isinstance(field, Field) and field.type == 'upload'\
8137 and fn is True:
8138 fn = field.uploadfield = '%s_blob' % field.name
8139 if isinstance(fn,str) and not fn in uploadfields:
8140 fields.append(Field(fn,'blob',default='',
8141 writable=False,readable=False))
8142
8143 lower_fieldnames = set()
8144 reserved = dir(Table) + ['fields']
8145 for field in fields:
8146 field_name = field.name
8147 if db and db.check_reserved:
8148 db.check_reserved_keyword(field_name)
8149 elif field_name in reserved:
8150 raise SyntaxError("field name %s not allowed" % field_name)
8151
8152 if field_name.lower() in lower_fieldnames:
8153 raise SyntaxError("duplicate field %s in table %s" \
8154 % (field_name, tablename))
8155 else:
8156 lower_fieldnames.add(field_name.lower())
8157
8158 self.fields.append(field_name)
8159 self[field_name] = field
8160 if field.type == 'id':
8161 self['id'] = field
8162 field.tablename = field._tablename = tablename
8163 field.table = field._table = self
8164 field.db = field._db = db
8165 if db and not field.type in ('text', 'blob', 'json') and \
8166 db._adapter.maxcharlength < field.length:
8167 field.length = db._adapter.maxcharlength
8168 self.ALL = SQLALL(self)
8169
8170 if hasattr(self,'_primarykey'):
8171 for k in self._primarykey:
8172 if k not in self.fields:
8173 raise SyntaxError(
8174 "primarykey must be a list of fields from table '%s " % tablename)
8175 else:
8176 self[k].notnull = True
8177 for field in virtual_fields:
8178 self[field.name] = field
8179
8180 @property
8183
8184 - def update(self,*args,**kwargs):
8185 raise RuntimeError("Syntax Not Supported")
8186
8187 - def _enable_record_versioning(self,
8188 archive_db=None,
8189 archive_name = '%(tablename)s_archive',
8190 current_record = 'current_record',
8191 is_active = 'is_active'):
8192 archive_db = archive_db or self._db
8193 archive_name = archive_name % dict(tablename=self._tablename)
8194 if archive_name in archive_db.tables():
8195 return
8196 fieldnames = self.fields()
8197 field_type = self if archive_db is self._db else 'bigint'
8198 archive_db.define_table(
8199 archive_name,
8200 Field(current_record,field_type),
8201 *[field.clone(unique=False) for field in self])
8202 self._before_update.append(
8203 lambda qset,fs,db=archive_db,an=archive_name,cn=current_record:
8204 archive_record(qset,fs,db[an],cn))
8205 if is_active and is_active in fieldnames:
8206 self._before_delete.append(
8207 lambda qset: qset.update(is_active=False))
8208 newquery = lambda query, t=self: t.is_active == True
8209 query = self._common_filter
8210 if query:
8211 newquery = query & newquery
8212 self._common_filter = newquery
8213
8221
8223 db = self._db
8224 pr = db._pending_references
8225 self._referenced_by = []
8226 for field in self:
8227 fieldname = field.name
8228 field_type = field.type
8229 if isinstance(field_type,str) and field_type[:10] == 'reference ':
8230 ref = field_type[10:].strip()
8231 if not ref.split():
8232 raise SyntaxError('Table: reference to nothing: %s' %ref)
8233 refs = ref.split('.')
8234 rtablename = refs[0]
8235 if not rtablename in db:
8236 pr[rtablename] = pr.get(rtablename,[]) + [field]
8237 continue
8238 rtable = db[rtablename]
8239 if len(refs)==2:
8240 rfieldname = refs[1]
8241 if not hasattr(rtable,'_primarykey'):
8242 raise SyntaxError(
8243 'keyed tables can only reference other keyed tables (for now)')
8244 if rfieldname not in rtable.fields:
8245 raise SyntaxError(
8246 "invalid field '%s' for referenced table '%s' in table '%s'" \
8247 % (rfieldname, rtablename, self._tablename))
8248 rtable._referenced_by.append(field)
8249 for referee in pr.get(self._tablename,[]):
8250 self._referenced_by.append(referee)
8251
8253 return dict([(k, v) for (k, v) in record.iteritems() if k
8254 in self.fields and (self[k].type!='id' or id)])
8255
8257 """ for keyed table only """
8258 query = None
8259 for k,v in key.iteritems():
8260 if k in self._primarykey:
8261 if query:
8262 query = query & (self[k] == v)
8263 else:
8264 query = (self[k] == v)
8265 else:
8266 raise SyntaxError(
8267 'Field %s is not part of the primary key of %s' % \
8268 (k,self._tablename))
8269 return query
8270
8272 if not key:
8273 return None
8274 elif isinstance(key, dict):
8275 """ for keyed table """
8276 query = self._build_query(key)
8277 rows = self._db(query).select()
8278 if rows:
8279 return rows[0]
8280 return None
8281 elif str(key).isdigit() or 'google' in DRIVERS and isinstance(key, Key):
8282 return self._db(self._id == key).select(limitby=(0,1)).first()
8283 elif key:
8284 return ogetattr(self, str(key))
8285
8287 for_update = kwargs.get('_for_update',False)
8288 if '_for_update' in kwargs: del kwargs['_for_update']
8289
8290 orderby = kwargs.get('_orderby',None)
8291 if '_orderby' in kwargs: del kwargs['_orderby']
8292
8293 if not key is DEFAULT:
8294 if isinstance(key, Query):
8295 record = self._db(key).select(
8296 limitby=(0,1),for_update=for_update, orderby=orderby).first()
8297 elif not str(key).isdigit():
8298 record = None
8299 else:
8300 record = self._db(self._id == key).select(
8301 limitby=(0,1),for_update=for_update, orderby=orderby).first()
8302 if record:
8303 for k,v in kwargs.iteritems():
8304 if record[k]!=v: return None
8305 return record
8306 elif kwargs:
8307 query = reduce(lambda a,b:a&b,[self[k]==v for k,v in kwargs.iteritems()])
8308 return self._db(query).select(limitby=(0,1),for_update=for_update, orderby=orderby).first()
8309 else:
8310 return None
8311
8313 if isinstance(key, dict) and isinstance(value, dict):
8314 """ option for keyed table """
8315 if set(key.keys()) == set(self._primarykey):
8316 value = self._filter_fields(value)
8317 kv = {}
8318 kv.update(value)
8319 kv.update(key)
8320 if not self.insert(**kv):
8321 query = self._build_query(key)
8322 self._db(query).update(**self._filter_fields(value))
8323 else:
8324 raise SyntaxError(
8325 'key must have all fields from primary key: %s'%\
8326 (self._primarykey))
8327 elif str(key).isdigit():
8328 if key == 0:
8329 self.insert(**self._filter_fields(value))
8330 elif self._db(self._id == key)\
8331 .update(**self._filter_fields(value)) is None:
8332 raise SyntaxError('No such record: %s' % key)
8333 else:
8334 if isinstance(key, dict):
8335 raise SyntaxError(
8336 'value must be a dictionary: %s' % value)
8337 osetattr(self, str(key), value)
8338
8339 __getattr__ = __getitem__
8340
8342 if key[:1]!='_' and key in self:
8343 raise SyntaxError('Object exists and cannot be redefined: %s' % key)
8344 osetattr(self,key,value)
8345
8347 if isinstance(key, dict):
8348 query = self._build_query(key)
8349 if not self._db(query).delete():
8350 raise SyntaxError('No such record: %s' % key)
8351 elif not str(key).isdigit() or \
8352 not self._db(self._id == key).delete():
8353 raise SyntaxError('No such record: %s' % key)
8354
8356 return hasattr(self,key)
8357
8358 has_key = __contains__
8359
8361 return self.__dict__.items()
8362
8364 for fieldname in self.fields:
8365 yield self[fieldname]
8366
8369
8370
8372 return '<Table %s (%s)>' % (self._tablename,','.join(self.fields()))
8373
8375 if hasattr(self,'_ot') and self._ot is not None:
8376 if 'Oracle' in str(type(self._db._adapter)):
8377 return '%s %s' % (self._ot, self._tablename)
8378 return '%s AS %s' % (self._ot, self._tablename)
8379 return self._tablename
8380
8381 - def _drop(self, mode = ''):
8382 return self._db._adapter._drop(self, mode)
8383
8384 - def drop(self, mode = ''):
8385 return self._db._adapter.drop(self,mode)
8386
8387 - def _listify(self,fields,update=False):
8388 new_fields = {}
8389
8390
8391 for name in fields:
8392 if not name in self.fields:
8393 if name != 'id':
8394 raise SyntaxError(
8395 'Field %s does not belong to the table' % name)
8396 else:
8397 field = self[name]
8398 value = fields[name]
8399 if field.filter_in:
8400 value = field.filter_in(value)
8401 new_fields[name] = (field,value)
8402
8403
8404 to_compute = []
8405 for ofield in self:
8406 name = ofield.name
8407 if not name in new_fields:
8408
8409 if ofield.compute:
8410 to_compute.append((name,ofield))
8411
8412 elif not update and not ofield.default is None:
8413 value = ofield.default
8414 fields[name] = value
8415 new_fields[name] = (ofield,value)
8416
8417 elif update and not ofield.update is None:
8418 value = ofield.update
8419 fields[name] = value
8420 new_fields[name] = (ofield,value)
8421
8422 elif not update and ofield.required:
8423 raise RuntimeError(
8424 'Table: missing required field: %s' % name)
8425
8426 if to_compute:
8427 row = Row(fields)
8428 for name,ofield in to_compute:
8429
8430 try:
8431 new_fields[name] = (ofield,ofield.compute(row))
8432 except (KeyError, AttributeError):
8433
8434 if ofield.required:
8435 raise SyntaxError('unable to comput field: %s' % name)
8436 return new_fields.values()
8437
8439 for field in self:
8440 if field.type=='upload' and field.name in fields:
8441 value = fields[field.name]
8442 if value and not isinstance(value,str):
8443 if hasattr(value,'file') and hasattr(value,'filename'):
8444 new_name = field.store(value.file,filename=value.filename)
8445 elif hasattr(value,'read') and hasattr(value,'name'):
8446 new_name = field.store(value,filename=value.name)
8447 else:
8448 raise RuntimeError("Unable to handle upload")
8449 fields[field.name] = new_name
8450
8452 "If there are no fields/values specified, return table defaults"
8453 if not fields:
8454 fields = {}
8455 for field in self:
8456 if field.type != "id":
8457 fields[field.name] = field.default
8458 return fields
8459
8463
8473
8489
8491 if _key is DEFAULT:
8492 record = self(**values)
8493 elif isinstance(_key,dict):
8494 record = self(**_key)
8495 else:
8496 record = self(_key)
8497 if record:
8498 record.update_record(**values)
8499 newid = None
8500 else:
8501 newid = self.insert(**values)
8502 return newid
8503
8505 """
8506 here items is a list of dictionaries
8507 """
8508 items = [self._listify(item) for item in items]
8509 if any(f(item) for item in items for f in self._before_insert):return 0
8510 ret = self._db._adapter.bulk_insert(self,items)
8511 ret and [[f(item,ret[k]) for k,item in enumerate(items)] for f in self._after_insert]
8512 return ret
8513
8515 return self._db._adapter._truncate(self, mode)
8516
8518 return self._db._adapter.truncate(self, mode)
8519
8520 - def import_from_csv_file(
8521 self,
8522 csvfile,
8523 id_map=None,
8524 null='<NULL>',
8525 unique='uuid',
8526 id_offset=None,
8527 *args, **kwargs
8528 ):
8529 """
8530 Import records from csv file.
8531 Column headers must have same names as table fields.
8532 Field 'id' is ignored.
8533 If column names read 'table.file' the 'table.' prefix is ignored.
8534 'unique' argument is a field which must be unique
8535 (typically a uuid field)
8536 'restore' argument is default False;
8537 if set True will remove old values in table first.
8538 'id_map' ff set to None will not map ids.
8539 The import will keep the id numbers in the restored table.
8540 This assumes that there is an field of type id that
8541 is integer and in incrementing order.
8542 Will keep the id numbers in restored table.
8543 """
8544
8545 delimiter = kwargs.get('delimiter', ',')
8546 quotechar = kwargs.get('quotechar', '"')
8547 quoting = kwargs.get('quoting', csv.QUOTE_MINIMAL)
8548 restore = kwargs.get('restore', False)
8549 if restore:
8550 self._db[self].truncate()
8551
8552 reader = csv.reader(csvfile, delimiter=delimiter,
8553 quotechar=quotechar, quoting=quoting)
8554 colnames = None
8555 if isinstance(id_map, dict):
8556 if not self._tablename in id_map:
8557 id_map[self._tablename] = {}
8558 id_map_self = id_map[self._tablename]
8559
8560 def fix(field, value, id_map, id_offset):
8561 list_reference_s='list:reference'
8562 if value == null:
8563 value = None
8564 elif field.type=='blob':
8565 value = base64.b64decode(value)
8566 elif field.type=='double' or field.type=='float':
8567 if not value.strip():
8568 value = None
8569 else:
8570 value = float(value)
8571 elif field.type in ('integer','bigint'):
8572 if not value.strip():
8573 value = None
8574 else:
8575 value = int(value)
8576 elif field.type.startswith('list:string'):
8577 value = bar_decode_string(value)
8578 elif field.type.startswith(list_reference_s):
8579 ref_table = field.type[len(list_reference_s):].strip()
8580 if id_map is not None:
8581 value = [id_map[ref_table][int(v)] \
8582 for v in bar_decode_string(value)]
8583 else:
8584 value = [v for v in bar_decode_string(value)]
8585 elif field.type.startswith('list:'):
8586 value = bar_decode_integer(value)
8587 elif id_map and field.type.startswith('reference'):
8588 try:
8589 value = id_map[field.type[9:].strip()][int(value)]
8590 except KeyError:
8591 pass
8592 elif id_offset and field.type.startswith('reference'):
8593 try:
8594 value = id_offset[field.type[9:].strip()]+int(value)
8595 except KeyError:
8596 pass
8597 return (field.name, value)
8598
8599 def is_id(colname):
8600 if colname in self:
8601 return self[colname].type == 'id'
8602 else:
8603 return False
8604
8605 first = True
8606 unique_idx = None
8607 for line in reader:
8608 if not line:
8609 break
8610 if not colnames:
8611 colnames = [x.split('.',1)[-1] for x in line][:len(line)]
8612 cols, cid = [], None
8613 for i,colname in enumerate(colnames):
8614 if is_id(colname):
8615 cid = i
8616 else:
8617 cols.append(i)
8618 if colname == unique:
8619 unique_idx = i
8620 else:
8621 items = [fix(self[colnames[i]], line[i], id_map, id_offset) \
8622 for i in cols if colnames[i] in self.fields]
8623
8624 if not id_map and cid is not None and id_offset is not None and not unique_idx:
8625 csv_id = int(line[cid])
8626 curr_id = self.insert(**dict(items))
8627 if first:
8628 first = False
8629
8630
8631
8632 if curr_id>csv_id:
8633 id_offset[self._tablename] = curr_id-csv_id
8634 else:
8635 id_offset[self._tablename] = 0
8636
8637 while curr_id<csv_id+id_offset[self._tablename]:
8638 self._db(self._db[self][colnames[cid]] == curr_id).delete()
8639 curr_id = self.insert(**dict(items))
8640
8641
8642 elif not unique_idx:
8643 new_id = self.insert(**dict(items))
8644 else:
8645 unique_value = line[unique_idx]
8646 query = self._db[self][unique] == unique_value
8647 record = self._db(query).select().first()
8648 if record:
8649 record.update_record(**dict(items))
8650 new_id = record[self._id.name]
8651 else:
8652 new_id = self.insert(**dict(items))
8653 if id_map and cid is not None:
8654 id_map_self[int(line[cid])] = new_id
8655
8656 - def as_dict(self, flat=False, sanitize=True, field_options=True):
8657 tablename = str(self)
8658 table_as_dict = dict(name=tablename, items={}, fields=[],
8659 sequence_name=self._sequence_name,
8660 trigger_name=self._trigger_name,
8661 common_filter=self._common_filter, format=self._format,
8662 singular=self._singular, plural=self._plural)
8663
8664 for field in self:
8665 if (field.readable or field.writable) or (not sanitize):
8666 table_as_dict["fields"].append(field.name)
8667 table_as_dict["items"][field.name] = \
8668 field.as_dict(flat=flat, sanitize=sanitize,
8669 options=field_options)
8670 return table_as_dict
8671
8672 - def as_xml(self, sanitize=True, field_options=True):
8678
8679 - def as_json(self, sanitize=True, field_options=True):
8685
8686 - def as_yaml(self, sanitize=True, field_options=True):
8692
8695
8696 - def on(self, query):
8697 return Expression(self._db,self._db._adapter.ON,self,query)
8698
8700 tablenames = qset.db._adapter.tables(qset.query)
8701 if len(tablenames)!=1: raise RuntimeError("cannot update join")
8702 table = qset.db[tablenames[0]]
8703 for row in qset.select():
8704 fields = archive_table._filter_fields(row)
8705 fields[current_record] = row.id
8706 archive_table.insert(**fields)
8707 return False
8708
8712
8713 - def __init__(
8714 self,
8715 db,
8716 op,
8717 first=None,
8718 second=None,
8719 type=None,
8720 **optional_args
8721 ):
8722
8723 self.db = db
8724 self.op = op
8725 self.first = first
8726 self.second = second
8727 self._table = getattr(first,'_table',None)
8728
8729 if not type and first and hasattr(first,'type'):
8730 self.type = first.type
8731 else:
8732 self.type = type
8733 self.optional_args = optional_args
8734
8738
8742
8746
8750
8754
8758
8762
8766
8770
8774
8778
8782
8786
8790
8794
8798
8802
8804 db = self.db
8805 if start < 0:
8806 pos0 = '(%s - %d)' % (self.len(), abs(start) - 1)
8807 else:
8808 pos0 = start + 1
8809
8810 if stop < 0:
8811 length = '(%s - %d - %s)' % (self.len(), abs(stop) - 1, pos0)
8812 elif stop == sys.maxint:
8813 length = self.len()
8814 else:
8815 length = '(%s - %s)' % (stop + 1, pos0)
8816 return Expression(db,db._adapter.SUBSTRING,
8817 self, (pos0, length), self.type)
8818
8820 return self[i:i + 1]
8821
8823 return self.db._adapter.expand(self,self.type)
8824
8826 db = self.db
8827 return Expression(db,db._adapter.COMMA,self,other,self.type)
8828
8834
8838
8840 db = self.db
8841 if self.type in ('integer','bigint'):
8842 result_type = 'integer'
8843 elif self.type in ['date','time','datetime','double','float']:
8844 result_type = 'double'
8845 elif self.type.startswith('decimal('):
8846 result_type = self.type
8847 else:
8848 raise SyntaxError("subtraction operation not supported for type")
8849 return Expression(db,db._adapter.SUB,self,other,result_type)
8850
8854
8858
8862
8866
8870
8874
8878
8882
8886
8887 - def like(self, value, case_sensitive=False):
8888 db = self.db
8889 op = case_sensitive and db._adapter.LIKE or db._adapter.ILIKE
8890 return Query(db, op, self, value)
8891
8895
8897 """
8898 Accepts the following inputs:
8899 field.belongs(1,2)
8900 field.belongs((1,2))
8901 field.belongs(query)
8902
8903 Does NOT accept:
8904 field.belongs(1)
8905 """
8906 db = self.db
8907 if len(value) == 1:
8908 value = value[0]
8909 if isinstance(value,Query):
8910 value = db(value)._select(value.first._table._id)
8911 return Query(db, db._adapter.BELONGS, self, value)
8912
8914 db = self.db
8915 if not self.type in ('string', 'text', 'json'):
8916 raise SyntaxError("startswith used with incompatible field type")
8917 return Query(db, db._adapter.STARTSWITH, self, value)
8918
8920 db = self.db
8921 if not self.type in ('string', 'text', 'json'):
8922 raise SyntaxError("endswith used with incompatible field type")
8923 return Query(db, db._adapter.ENDSWITH, self, value)
8924
8925 - def contains(self, value, all=False, case_sensitive=False):
8926 """
8927 The case_sensitive parameters is only useful for PostgreSQL
8928 For other RDMBs it is ignored and contains is always case in-sensitive
8929 For MongoDB and GAE contains is always case sensitive
8930 """
8931 db = self.db
8932 if isinstance(value,(list, tuple)):
8933 subqueries = [self.contains(str(v).strip(),case_sensitive=case_sensitive)
8934 for v in value if str(v).strip()]
8935 if not subqueries:
8936 return self.contains('')
8937 else:
8938 return reduce(all and AND or OR,subqueries)
8939 if not self.type in ('string', 'text', 'json') and not self.type.startswith('list:'):
8940 raise SyntaxError("contains used with incompatible field type")
8941 return Query(db, db._adapter.CONTAINS, self, value, case_sensitive=case_sensitive)
8942
8946
8947
8948
8949 - def st_asgeojson(self, precision=15, options=0, version=1):
8953
8954 - def st_astext(self):
8955 db = self.db
8956 return Expression(db, db._adapter.ST_ASTEXT, self, type='string')
8957
8961
8965
8969
8973
8974
8975
8979
8983
8987
8991
8995
8999
9004 """
9005 allows defining of custom SQL types
9006
9007 Example::
9008
9009 decimal = SQLCustomType(
9010 type ='double',
9011 native ='integer',
9012 encoder =(lambda x: int(float(x) * 100)),
9013 decoder = (lambda x: Decimal("0.00") + Decimal(str(float(x)/100)) )
9014 )
9015
9016 db.define_table(
9017 'example',
9018 Field('value', type=decimal)
9019 )
9020
9021 :param type: the web2py type (default = 'string')
9022 :param native: the backend type
9023 :param encoder: how to encode the value to store it in the backend
9024 :param decoder: how to decode the value retrieved from the backend
9025 :param validator: what validators to use ( default = None, will use the
9026 default validator for type)
9027 """
9028
9029 - def __init__(
9030 self,
9031 type='string',
9032 native=None,
9033 encoder=None,
9034 decoder=None,
9035 validator=None,
9036 _class=None,
9037 ):
9038
9039 self.type = type
9040 self.native = native
9041 self.encoder = encoder or (lambda x: x)
9042 self.decoder = decoder or (lambda x: x)
9043 self.validator = validator
9044 self._class = _class or type
9045
9047 try:
9048 return self.type.startswith(self, text)
9049 except TypeError:
9050 return False
9051
9054
9057
9060
9062 - def __init__(self, name, f=None, ftype='string',label=None,table_name=None):
9076
9078 - def __init__(self, name, f=None, handler=None):
9082
9084 return ', '.join(str(y) for y in x or [])
9085
9086 -class Field(Expression):
9087
9088 Virtual = FieldVirtual
9089 Method = FieldMethod
9090 Lazy = FieldMethod
9091
9092 """
9093 an instance of this class represents a database field
9094
9095 example::
9096
9097 a = Field(name, 'string', length=32, default=None, required=False,
9098 requires=IS_NOT_EMPTY(), ondelete='CASCADE',
9099 notnull=False, unique=False,
9100 uploadfield=True, widget=None, label=None, comment=None,
9101 uploadfield=True, # True means store on disk,
9102 # 'a_field_name' means store in this field in db
9103 # False means file content will be discarded.
9104 writable=True, readable=True, update=None, authorize=None,
9105 autodelete=False, represent=None, uploadfolder=None,
9106 uploadseparate=False # upload to separate directories by uuid_keys
9107 # first 2 character and tablename.fieldname
9108 # False - old behavior
9109 # True - put uploaded file in
9110 # <uploaddir>/<tablename>.<fieldname>/uuid_key[:2]
9111 # directory)
9112 uploadfs=None # a pyfilesystem where to store upload
9113
9114 to be used as argument of DAL.define_table
9115
9116 allowed field types:
9117 string, boolean, integer, double, text, blob,
9118 date, time, datetime, upload, password
9119
9120 strings must have a length of Adapter.maxcharlength by default (512 or 255 for mysql)
9121 fields should have a default or they will be required in SQLFORMs
9122 the requires argument is used to validate the field input in SQLFORMs
9123
9124 """
9125
9126 - def __init__(
9127 self,
9128 fieldname,
9129 type='string',
9130 length=None,
9131 default=DEFAULT,
9132 required=False,
9133 requires=DEFAULT,
9134 ondelete='CASCADE',
9135 notnull=False,
9136 unique=False,
9137 uploadfield=True,
9138 widget=None,
9139 label=None,
9140 comment=None,
9141 writable=True,
9142 readable=True,
9143 update=None,
9144 authorize=None,
9145 autodelete=False,
9146 represent=None,
9147 uploadfolder=None,
9148 uploadseparate=False,
9149 uploadfs=None,
9150 compute=None,
9151 custom_store=None,
9152 custom_retrieve=None,
9153 custom_retrieve_file_properties=None,
9154 custom_delete=None,
9155 filter_in = None,
9156 filter_out = None,
9157 custom_qualifier = None,
9158 map_none = None,
9159 ):
9160 self._db = self.db = None
9161 self.op = None
9162 self.first = None
9163 self.second = None
9164 self.name = fieldname = cleanup(fieldname)
9165 if not isinstance(fieldname,str) or hasattr(Table,fieldname) or \
9166 fieldname[0] == '_' or REGEX_PYTHON_KEYWORDS.match(fieldname):
9167 raise SyntaxError('Field: invalid field name: %s' % fieldname)
9168 self.type = type if not isinstance(type, (Table,Field)) else 'reference %s' % type
9169 self.length = length if not length is None else DEFAULTLENGTH.get(self.type,512)
9170 self.default = default if default!=DEFAULT else (update or None)
9171 self.required = required
9172 self.ondelete = ondelete.upper()
9173 self.notnull = notnull
9174 self.unique = unique
9175 self.uploadfield = uploadfield
9176 self.uploadfolder = uploadfolder
9177 self.uploadseparate = uploadseparate
9178 self.uploadfs = uploadfs
9179 self.widget = widget
9180 self.comment = comment
9181 self.writable = writable
9182 self.readable = readable
9183 self.update = update
9184 self.authorize = authorize
9185 self.autodelete = autodelete
9186 self.represent = list_represent if \
9187 represent==None and type in ('list:integer','list:string') else represent
9188 self.compute = compute
9189 self.isattachment = True
9190 self.custom_store = custom_store
9191 self.custom_retrieve = custom_retrieve
9192 self.custom_retrieve_file_properties = custom_retrieve_file_properties
9193 self.custom_delete = custom_delete
9194 self.filter_in = filter_in
9195 self.filter_out = filter_out
9196 self.custom_qualifier = custom_qualifier
9197 self.label = label if label!=None else fieldname.replace('_',' ').title()
9198 self.requires = requires if requires!=None else []
9199 self.map_none = map_none
9200
9202 self.__dict__.update(*args,**attributes)
9203
9204 - def clone(self,point_self_references_to=False,**args):
9205 field = copy.copy(self)
9206 if point_self_references_to and \
9207 field.type == 'reference %s'+field._tablename:
9208 field.type = 'reference %s' % point_self_references_to
9209 field.__dict__.update(args)
9210 return field
9211
9212 - def store(self, file, filename=None, path=None):
9213 if self.custom_store:
9214 return self.custom_store(file,filename,path)
9215 if isinstance(file, cgi.FieldStorage):
9216 filename = filename or file.filename
9217 file = file.file
9218 elif not filename:
9219 filename = file.name
9220 filename = os.path.basename(filename.replace('/', os.sep)\
9221 .replace('\\', os.sep))
9222 m = REGEX_STORE_PATTERN.search(filename)
9223 extension = m and m.group('e') or 'txt'
9224 uuid_key = web2py_uuid().replace('-', '')[-16:]
9225 encoded_filename = base64.b16encode(filename).lower()
9226 newfilename = '%s.%s.%s.%s' % \
9227 (self._tablename, self.name, uuid_key, encoded_filename)
9228 newfilename = newfilename[:(self.length - 1 - len(extension))] + '.' + extension
9229 self_uploadfield = self.uploadfield
9230 if isinstance(self_uploadfield,Field):
9231 blob_uploadfield_name = self_uploadfield.uploadfield
9232 keys={self_uploadfield.name: newfilename,
9233 blob_uploadfield_name: file.read()}
9234 self_uploadfield.table.insert(**keys)
9235 elif self_uploadfield == True:
9236 if path:
9237 pass
9238 elif self.uploadfolder:
9239 path = self.uploadfolder
9240 elif self.db._adapter.folder:
9241 path = pjoin(self.db._adapter.folder, '..', 'uploads')
9242 else:
9243 raise RuntimeError(
9244 "you must specify a Field(...,uploadfolder=...)")
9245 if self.uploadseparate:
9246 if self.uploadfs:
9247 raise RuntimeError("not supported")
9248 path = pjoin(path,"%s.%s" %(self._tablename, self.name),
9249 uuid_key[:2])
9250 if not exists(path):
9251 os.makedirs(path)
9252 pathfilename = pjoin(path, newfilename)
9253 if self.uploadfs:
9254 dest_file = self.uploadfs.open(newfilename, 'wb')
9255 else:
9256 dest_file = open(pathfilename, 'wb')
9257 try:
9258 shutil.copyfileobj(file, dest_file)
9259 except IOError:
9260 raise IOError(
9261 'Unable to store file "%s" because invalid permissions, readonly file system, or filename too long' % pathfilename)
9262 dest_file.close()
9263 return newfilename
9264
9265 - def retrieve(self, name, path=None, nameonly=False):
9266 """
9267 if nameonly==True return (filename, fullfilename) instead of
9268 (filename, stream)
9269 """
9270 self_uploadfield = self.uploadfield
9271 if self.custom_retrieve:
9272 return self.custom_retrieve(name, path)
9273 import http
9274 if self.authorize or isinstance(self_uploadfield, str):
9275 row = self.db(self == name).select().first()
9276 if not row:
9277 raise http.HTTP(404)
9278 if self.authorize and not self.authorize(row):
9279 raise http.HTTP(403)
9280 m = REGEX_UPLOAD_PATTERN.match(name)
9281 if not m or not self.isattachment:
9282 raise TypeError('Can\'t retrieve %s' % name)
9283 file_properties = self.retrieve_file_properties(name,path)
9284 filename = file_properties['filename']
9285 if isinstance(self_uploadfield, str):
9286 stream = StringIO.StringIO(row[self_uploadfield] or '')
9287 elif isinstance(self_uploadfield,Field):
9288 blob_uploadfield_name = self_uploadfield.uploadfield
9289 query = self_uploadfield == name
9290 data = self_uploadfield.table(query)[blob_uploadfield_name]
9291 stream = StringIO.StringIO(data)
9292 elif self.uploadfs:
9293
9294 stream = self.uploadfs.open(name, 'rb')
9295 else:
9296
9297
9298
9299 fullname = pjoin(file_properties['path'],name)
9300 if nameonly:
9301 return (filename, fullname)
9302 stream = open(fullname,'rb')
9303 return (filename, stream)
9304
9306 self_uploadfield = self.uploadfield
9307 if self.custom_retrieve_file_properties:
9308 return self.custom_retrieve_file_properties(name, path)
9309 try:
9310 m = REGEX_UPLOAD_PATTERN.match(name)
9311 if not m or not self.isattachment:
9312 raise TypeError('Can\'t retrieve %s file properties' % name)
9313 filename = base64.b16decode(m.group('name'), True)
9314 filename = REGEX_CLEANUP_FN.sub('_', filename)
9315 except (TypeError, AttributeError):
9316 filename = name
9317 if isinstance(self_uploadfield, str):
9318 return dict(path=None,filename=filename)
9319 elif isinstance(self_uploadfield,Field):
9320 return dict(path=None,filename=filename)
9321 else:
9322
9323 if path:
9324 pass
9325 elif self.uploadfolder:
9326 path = self.uploadfolder
9327 else:
9328 path = pjoin(self.db._adapter.folder, '..', 'uploads')
9329 if self.uploadseparate:
9330 t = m.group('table')
9331 f = m.group('field')
9332 u = m.group('uuidkey')
9333 path = pjoin(path,"%s.%s" % (t,f),u[:2])
9334 return dict(path=path,filename=filename)
9335
9336
9352
9364
9365 - def count(self, distinct=None):
9367
9368 - def as_dict(self, flat=False, sanitize=True, options=True):
9369
9370 attrs = ('type', 'length', 'default', 'required',
9371 'ondelete', 'notnull', 'unique', 'uploadfield',
9372 'widget', 'label', 'comment', 'writable', 'readable',
9373 'update', 'authorize', 'autodelete', 'represent',
9374 'uploadfolder', 'uploadseparate', 'uploadfs',
9375 'compute', 'custom_store', 'custom_retrieve',
9376 'custom_retrieve_file_properties', 'custom_delete',
9377 'filter_in', 'filter_out', 'custom_qualifier',
9378 'map_none', 'name')
9379
9380 SERIALIZABLE_TYPES = (int, long, basestring, dict, list,
9381 float, tuple, bool, type(None))
9382
9383 def flatten(obj):
9384 if flat:
9385 if isinstance(obj, flatten.__class__):
9386 return str(type(obj))
9387 elif isinstance(obj, type):
9388 try:
9389 return str(obj).split("'")[1]
9390 except IndexError:
9391 return str(obj)
9392 elif not isinstance(obj, SERIALIZABLE_TYPES):
9393 return str(obj)
9394 elif isinstance(obj, dict):
9395 newobj = dict()
9396 for k, v in obj.items():
9397 newobj[k] = flatten(v)
9398 return newobj
9399 elif isinstance(obj, (list, tuple, set)):
9400 return [flatten(v) for v in obj]
9401 else:
9402 return obj
9403 elif isinstance(obj, (dict, set)):
9404 return obj.copy()
9405 else: return obj
9406
9407 def filter_requires(t, r, options=True):
9408 if sanitize and any([keyword in str(t).upper() for
9409 keyword in ("CRYPT", "IS_STRONG")]):
9410 return None
9411
9412 if not isinstance(r, dict):
9413 if options and hasattr(r, "options"):
9414 if callable(r.options):
9415 r.options()
9416 newr = r.__dict__.copy()
9417 else:
9418 newr = r.copy()
9419
9420
9421 if not options and newr.has_key("labels"):
9422 [newr.update({key:None}) for key in
9423 ("labels", "theset") if (key in newr)]
9424
9425 for k, v in newr.items():
9426 if k == "other":
9427 if isinstance(v, dict):
9428 otype, other = v.popitem()
9429 else:
9430 otype = flatten(type(v))
9431 other = v
9432 newr[k] = {otype: filter_requires(otype, other,
9433 options=options)}
9434 else:
9435 newr[k] = flatten(v)
9436 return newr
9437
9438 if isinstance(self.requires, (tuple, list, set)):
9439 requires = dict([(flatten(type(r)),
9440 filter_requires(type(r), r,
9441 options=options)) for
9442 r in self.requires])
9443 else:
9444 requires = {flatten(type(self.requires)):
9445 filter_requires(type(self.requires),
9446 self.requires, options=options)}
9447
9448 d = dict(colname="%s.%s" % (self.tablename, self.name),
9449 requires=requires)
9450 d.update([(attr, flatten(getattr(self, attr))) for attr in attrs])
9451 return d
9452
9453 - def as_xml(self, sanitize=True, options=True):
9461
9462 - def as_json(self, sanitize=True, options=True):
9470
9471 - def as_yaml(self, sanitize=True, options=True):
9478
9481
9483 try:
9484 return '%s.%s' % (self.tablename, self.name)
9485 except:
9486 return '<no table>.%s' % self.name
9487
9488
9489 -class Query(object):
9490
9491 """
9492 a query object necessary to define a set.
9493 it can be stored or can be passed to DAL.__call__() to obtain a Set
9494
9495 Example::
9496
9497 query = db.users.name=='Max'
9498 set = db(query)
9499 records = set.select()
9500
9501 """
9502
9503 - def __init__(
9504 self,
9505 db,
9506 op,
9507 first=None,
9508 second=None,
9509 ignore_common_filters = False,
9510 **optional_args
9511 ):
9512 self.db = self._db = db
9513 self.op = op
9514 self.first = first
9515 self.second = second
9516 self.ignore_common_filters = ignore_common_filters
9517 self.optional_args = optional_args
9518
9521
9523 return self.db._adapter.expand(self)
9524
9527
9529 return Query(self.db,self.db._adapter.OR,self,other)
9530
9532 if self.op==self.db._adapter.NOT:
9533 return self.first
9534 return Query(self.db,self.db._adapter.NOT,self)
9535
9537 return repr(self) == repr(other)
9538
9540 return not (self == other)
9541
9542 - def case(self,t=1,f=0):
9543 return self.db._adapter.CASE(self,t,f)
9544
9545 - def as_dict(self, flat=False, sanitize=True):
9546 """Experimental stuff
9547
9548 This allows to return a plain dictionary with the basic
9549 query representation. Can be used with json/xml services
9550 for client-side db I/O
9551
9552 Example:
9553 >>> q = db.auth_user.id != 0
9554 >>> q.as_dict(flat=True)
9555 {"op": "NE", "first":{"tablename": "auth_user",
9556 "fieldname": "id"},
9557 "second":0}
9558 """
9559
9560 SERIALIZABLE_TYPES = (tuple, dict, list, int, long, float,
9561 basestring, type(None), bool)
9562 def loop(d):
9563 newd = dict()
9564 for k, v in d.items():
9565 if k in ("first", "second"):
9566 if isinstance(v, self.__class__):
9567 newd[k] = loop(v.__dict__)
9568 elif isinstance(v, Field):
9569 newd[k] = {"tablename": v._tablename,
9570 "fieldname": v.name}
9571 elif isinstance(v, Expression):
9572 newd[k] = loop(v.__dict__)
9573 elif isinstance(v, SERIALIZABLE_TYPES):
9574 newd[k] = v
9575 elif isinstance(v, (datetime.date,
9576 datetime.time,
9577 datetime.datetime)):
9578 newd[k] = unicode(v)
9579 elif k == "op":
9580 if callable(v):
9581 newd[k] = v.__name__
9582 elif isinstance(v, basestring):
9583 newd[k] = v
9584 else: pass
9585 elif isinstance(v, SERIALIZABLE_TYPES):
9586 if isinstance(v, dict):
9587 newd[k] = loop(v)
9588 else: newd[k] = v
9589 return newd
9590
9591 if flat:
9592 return loop(self.__dict__)
9593 else: return self.__dict__
9594
9595
9596 - def as_xml(self, sanitize=True):
9603
9604 - def as_json(self, sanitize=True):
9611
9613 if not orderby:
9614 return None
9615 orderby2 = orderby[0]
9616 for item in orderby[1:]:
9617 orderby2 = orderby2 | item
9618 return orderby2
9619
9621 return (query and hasattr(query,'ignore_common_filters') and \
9622 not query.ignore_common_filters)
9623
9625
9626 """
9627 a Set represents a set of records in the database,
9628 the records are identified by the query=Query(...) object.
9629 normally the Set is generated by DAL.__call__(Query(...))
9630
9631 given a set, for example
9632 set = db(db.users.name=='Max')
9633 you can:
9634 set.update(db.users.name='Massimo')
9635 set.delete() # all elements in the set
9636 set.select(orderby=db.users.id, groupby=db.users.name, limitby=(0,10))
9637 and take subsets:
9638 subset = set(db.users.id<5)
9639 """
9640
9641 - def __init__(self, db, query, ignore_common_filters = None):
9642 self.db = db
9643 self._db = db
9644 self.dquery = None
9645
9646
9647 if isinstance(query, dict):
9648 query = self.parse(query)
9649
9650 if not ignore_common_filters is None and \
9651 use_common_filters(query) == ignore_common_filters:
9652 query = copy.copy(query)
9653 query.ignore_common_filters = ignore_common_filters
9654 self.query = query
9655
9658
9659 - def __call__(self, query, ignore_common_filters=False):
9660 if isinstance(query,Table):
9661 query = self.db._adapter.id_query(query)
9662 elif isinstance(query,str):
9663 query = Expression(self.db,query)
9664 elif isinstance(query,Field):
9665 query = query!=None
9666 if self.query:
9667 return Set(self.db, self.query & query,
9668 ignore_common_filters=ignore_common_filters)
9669 else:
9670 return Set(self.db, query,
9671 ignore_common_filters=ignore_common_filters)
9672
9673 - def _count(self,distinct=None):
9674 return self.db._adapter._count(self.query,distinct)
9675
9676 - def _select(self, *fields, **attributes):
9677 adapter = self.db._adapter
9678 tablenames = adapter.tables(self.query,
9679 attributes.get('join',None),
9680 attributes.get('left',None),
9681 attributes.get('orderby',None),
9682 attributes.get('groupby',None))
9683 fields = adapter.expand_all(fields, tablenames)
9684 return adapter._select(self.query,fields,attributes)
9685
9687 db = self.db
9688 tablename = db._adapter.get_table(self.query)
9689 return db._adapter._delete(tablename,self.query)
9690
9691 - def _update(self, **update_fields):
9696
9697 - def as_dict(self, flat=False, sanitize=True):
9698 if flat:
9699 uid = dbname = uri = None
9700 codec = self.db._db_codec
9701 if not sanitize:
9702 uri, dbname, uid = (self.db._dbname, str(self.db),
9703 self.db._db_uid)
9704 d = {"query": self.query.as_dict(flat=flat)}
9705 d["db"] = {"uid": uid, "codec": codec,
9706 "name": dbname, "uri": uri}
9707 return d
9708 else: return self.__dict__
9709
9710 - def as_xml(self, sanitize=True):
9717
9718 - def as_json(self, sanitize=True):
9725
9726 - def parse(self, dquery):
9727 "Experimental: Turn a dictionary into a Query object"
9728 self.dquery = dquery
9729 return self.build(self.dquery)
9730
9732 "Experimental: see .parse()"
9733 op, first, second = (d["op"], d["first"],
9734 d.get("second", None))
9735 left = right = built = None
9736
9737 if op in ("AND", "OR"):
9738 if not (type(first), type(second)) == (dict, dict):
9739 raise SyntaxError("Invalid AND/OR query")
9740 if op == "AND":
9741 built = self.build(first) & self.build(second)
9742 else: built = self.build(first) | self.build(second)
9743
9744 elif op == "NOT":
9745 if first is None:
9746 raise SyntaxError("Invalid NOT query")
9747 built = ~self.build(first)
9748 else:
9749
9750 for k, v in {"left": first, "right": second}.items():
9751 if isinstance(v, dict) and v.get("op"):
9752 v = self.build(v)
9753 if isinstance(v, dict) and ("tablename" in v):
9754 v = self.db[v["tablename"]][v["fieldname"]]
9755 if k == "left": left = v
9756 else: right = v
9757
9758 if hasattr(self.db._adapter, op):
9759 opm = getattr(self.db._adapter, op)
9760
9761 if op == "EQ": built = left == right
9762 elif op == "NE": built = left != right
9763 elif op == "GT": built = left > right
9764 elif op == "GE": built = left >= right
9765 elif op == "LT": built = left < right
9766 elif op == "LE": built = left <= right
9767 elif op in ("JOIN", "LEFT_JOIN", "RANDOM", "ALLOW_NULL"):
9768 built = Expression(self.db, opm)
9769 elif op in ("LOWER", "UPPER", "EPOCH", "PRIMARY_KEY",
9770 "COALESCE_ZERO", "RAW", "INVERT"):
9771 built = Expression(self.db, opm, left)
9772 elif op in ("COUNT", "EXTRACT", "AGGREGATE", "SUBSTRING",
9773 "REGEXP", "LIKE", "ILIKE", "STARTSWITH",
9774 "ENDSWITH", "ADD", "SUB", "MUL", "DIV",
9775 "MOD", "AS", "ON", "COMMA", "NOT_NULL",
9776 "COALESCE", "CONTAINS", "BELONGS"):
9777 built = Expression(self.db, opm, left, right)
9778
9779 elif not (left or right): built = Expression(self.db, op)
9780 else:
9781 raise SyntaxError("Operator not supported: %s" % op)
9782
9783 return built
9784
9786 return not self.select(limitby=(0,1))
9787
9788 - def count(self,distinct=None, cache=None):
9789 db = self.db
9790 if cache:
9791 cache_model, time_expire = cache
9792 sql = self._count(distinct=distinct)
9793 key = db._uri + '/' + sql
9794 if len(key)>200: key = hashlib_md5(key).hexdigest()
9795 return cache_model(
9796 key,
9797 (lambda self=self,distinct=distinct: \
9798 db._adapter.count(self.query,distinct)),
9799 time_expire)
9800 return db._adapter.count(self.query,distinct)
9801
9802 - def select(self, *fields, **attributes):
9803 adapter = self.db._adapter
9804 tablenames = adapter.tables(self.query,
9805 attributes.get('join',None),
9806 attributes.get('left',None),
9807 attributes.get('orderby',None),
9808 attributes.get('groupby',None))
9809 fields = adapter.expand_all(fields, tablenames)
9810 return adapter.select(self.query,fields,attributes)
9811
9814
9816 db = self.db
9817 tablename = db._adapter.get_table(self.query)
9818 table = db[tablename]
9819 if any(f(self) for f in table._before_delete): return 0
9820 ret = db._adapter.delete(tablename,self.query)
9821 ret and [f(self) for f in table._after_delete]
9822 return ret
9823
9824 - def update(self, **update_fields):
9825 db = self.db
9826 tablename = db._adapter.get_table(self.query)
9827 table = db[tablename]
9828 table._attempt_upload(update_fields)
9829 if any(f(self,update_fields) for f in table._before_update):
9830 return 0
9831 fields = table._listify(update_fields,update=True)
9832 if not fields:
9833 raise SyntaxError("No fields to update")
9834 ret = db._adapter.update(tablename,self.query,fields)
9835 ret and [f(self,update_fields) for f in table._after_update]
9836 return ret
9837
9839 """
9840 same as update but does not call table._before_update and _after_update
9841 """
9842 tablename = self.db._adapter.get_table(self.query)
9843 table = self.db[tablename]
9844 fields = table._listify(update_fields,update=True)
9845 if not fields: raise SyntaxError("No fields to update")
9846 ret = self.db._adapter.update(tablename,self.query,fields)
9847 return ret
9848
9850 tablename = self.db._adapter.get_table(self.query)
9851 response = Row()
9852 response.errors = Row()
9853 new_fields = copy.copy(update_fields)
9854 for key,value in update_fields.iteritems():
9855 value,error = self.db[tablename][key].validate(value)
9856 if error:
9857 response.errors[key] = error
9858 else:
9859 new_fields[key] = value
9860 table = self.db[tablename]
9861 if response.errors:
9862 response.updated = None
9863 else:
9864 if not any(f(self,new_fields) for f in table._before_update):
9865 fields = table._listify(new_fields,update=True)
9866 if not fields: raise SyntaxError("No fields to update")
9867 ret = self.db._adapter.update(tablename,self.query,fields)
9868 ret and [f(self,new_fields) for f in table._after_update]
9869 else:
9870 ret = 0
9871 response.updated = ret
9872 return response
9873
9875 table = self.db[self.db._adapter.tables(self.query)[0]]
9876
9877 if upload_fields:
9878 fields = upload_fields.keys()
9879 else:
9880 fields = table.fields
9881 fields = [f for f in fields if table[f].type == 'upload'
9882 and table[f].uploadfield == True
9883 and table[f].autodelete]
9884 if not fields:
9885 return False
9886 for record in self.select(*[table[f] for f in fields]):
9887 for fieldname in fields:
9888 field = table[fieldname]
9889 oldname = record.get(fieldname, None)
9890 if not oldname:
9891 continue
9892 if upload_fields and oldname == upload_fields[fieldname]:
9893 continue
9894 if field.custom_delete:
9895 field.custom_delete(oldname)
9896 else:
9897 uploadfolder = field.uploadfolder
9898 if not uploadfolder:
9899 uploadfolder = pjoin(
9900 self.db._adapter.folder, '..', 'uploads')
9901 if field.uploadseparate:
9902 items = oldname.split('.')
9903 uploadfolder = pjoin(
9904 uploadfolder,
9905 "%s.%s" % (items[0], items[1]),
9906 items[2][:2])
9907 oldpath = pjoin(uploadfolder, oldname)
9908 if exists(oldpath):
9909 os.unlink(oldpath)
9910 return False
9911
9913 - def __init__(self, colset, table, id):
9914 self.colset, self.db, self.tablename, self.id = \
9915 colset, table._db, table._tablename, id
9916
9918 colset, db, tablename, id = self.colset, self.db, self.tablename, self.id
9919 table = db[tablename]
9920 newfields = fields or dict(colset)
9921 for fieldname in newfields.keys():
9922 if not fieldname in table.fields or table[fieldname].type=='id':
9923 del newfields[fieldname]
9924 table._db(table._id==id,ignore_common_filters=True).update(**newfields)
9925 colset.update(newfields)
9926 return colset
9927
9930 self.db, self.tablename, self.id = table._db, table._tablename, id
9932 return self.db(self.db[self.tablename]._id==self.id).delete()
9933
9936 self.db, self.tablename, self.fieldname, self.id = \
9937 field.db, field._tablename, field.name, id
9939 query = self.db[self.tablename][self.fieldname]==self.id
9940 return Set(self.db,query)
9943 - def __call__(self, query, ignore_common_filters=False):
9944 return self._getset()(query, ignore_common_filters)
9945 - def _count(self,distinct=None):
9947 - def _select(self, *fields, **attributes):
9951 - def _update(self, **update_fields):
9955 - def count(self,distinct=None, cache=None):
9957 - def select(self, *fields, **attributes):
9963 - def update(self, **update_fields):
9971
9974 self.method=method
9975 self.row=row
9977 return self.method(self.row,*args,**kwargs)
9978
9980 f.__lazy__ = True
9981 return f
9982
9983 -class Rows(object):
9984
9985 """
9986 A wrapper for the return value of a select. It basically represents a table.
9987 It has an iterator and each row is represented as a dictionary.
9988 """
9989
9990
9991
9992 - def __init__(
9993 self,
9994 db=None,
9995 records=[],
9996 colnames=[],
9997 compact=True,
9998 rawrows=None
9999 ):
10000 self.db = db
10001 self.records = records
10002 self.colnames = colnames
10003 self.compact = compact
10004 self.response = rawrows
10005
10007 return '<Rows (%s)>' % len(self.records)
10008
10010 """
10011 db.define_table('x',Field('number','integer'))
10012 if db(db.x).isempty(): [db.x.insert(number=i) for i in range(10)]
10013
10014 from gluon.dal import lazy_virtualfield
10015
10016 class MyVirtualFields(object):
10017 # normal virtual field (backward compatible, discouraged)
10018 def normal_shift(self): return self.x.number+1
10019 # lazy virtual field (because of @staticmethod)
10020 @lazy_virtualfield
10021 def lazy_shift(instance,row,delta=4): return row.x.number+delta
10022 db.x.virtualfields.append(MyVirtualFields())
10023
10024 for row in db(db.x).select():
10025 print row.number, row.normal_shift, row.lazy_shift(delta=7)
10026 """
10027 if not keyed_virtualfields:
10028 return self
10029 for row in self.records:
10030 for (tablename,virtualfields) in keyed_virtualfields.iteritems():
10031 attributes = dir(virtualfields)
10032 if not tablename in row:
10033 box = row[tablename] = Row()
10034 else:
10035 box = row[tablename]
10036 updated = False
10037 for attribute in attributes:
10038 if attribute[0] != '_':
10039 method = getattr(virtualfields,attribute)
10040 if hasattr(method,'__lazy__'):
10041 box[attribute]=VirtualCommand(method,row)
10042 elif type(method)==types.MethodType:
10043 if not updated:
10044 virtualfields.__dict__.update(row)
10045 updated = True
10046 box[attribute]=method()
10047 return self
10048
10050 if self.colnames!=other.colnames:
10051 raise Exception('Cannot & incompatible Rows objects')
10052 records = self.records+other.records
10053 return Rows(self.db,records,self.colnames)
10054
10056 if self.colnames!=other.colnames:
10057 raise Exception('Cannot | incompatible Rows objects')
10058 records = self.records
10059 records += [record for record in other.records \
10060 if not record in records]
10061 return Rows(self.db,records,self.colnames)
10062
10064 if len(self.records):
10065 return 1
10066 return 0
10067
10069 return len(self.records)
10070
10072 return Rows(self.db,self.records[a:b],self.colnames)
10073
10075 row = self.records[i]
10076 keys = row.keys()
10077 if self.compact and len(keys) == 1 and keys[0] != '_extra':
10078 return row[row.keys()[0]]
10079 return row
10080
10082 """
10083 iterator over records
10084 """
10085
10086 for i in xrange(len(self)):
10087 yield self[i]
10088
10090 """
10091 serializes the table into a csv file
10092 """
10093
10094 s = StringIO.StringIO()
10095 self.export_to_csv_file(s)
10096 return s.getvalue()
10097
10099 if not self.records:
10100 return None
10101 return self[0]
10102
10104 if not self.records:
10105 return None
10106 return self[-1]
10107
10108 - def find(self,f,limitby=None):
10109 """
10110 returns a new Rows object, a subset of the original object,
10111 filtered by the function f
10112 """
10113 if not self:
10114 return Rows(self.db, [], self.colnames)
10115 records = []
10116 if limitby:
10117 a,b = limitby
10118 else:
10119 a,b = 0,len(self)
10120 k = 0
10121 for row in self:
10122 if f(row):
10123 if a<=k: records.append(row)
10124 k += 1
10125 if k==b: break
10126 return Rows(self.db, records, self.colnames)
10127
10129 """
10130 removes elements from the calling Rows object, filtered by the function f,
10131 and returns a new Rows object containing the removed elements
10132 """
10133 if not self.records:
10134 return Rows(self.db, [], self.colnames)
10135 removed = []
10136 i=0
10137 while i<len(self):
10138 row = self[i]
10139 if f(row):
10140 removed.append(self.records[i])
10141 del self.records[i]
10142 else:
10143 i += 1
10144 return Rows(self.db, removed, self.colnames)
10145
10146 - def sort(self, f, reverse=False):
10147 """
10148 returns a list of sorted elements (not sorted in place)
10149 """
10150 rows = Rows(self.db,[],self.colnames,compact=False)
10151 rows.records = sorted(self,key=f,reverse=reverse)
10152 return rows
10153
10154
10156 """
10157 regroups the rows, by one of the fields
10158 """
10159 if not self.records:
10160 return {}
10161 key = str(field)
10162 grouped_row_group = dict()
10163
10164 for row in self:
10165 value = row[key]
10166 if not value in grouped_row_group:
10167 grouped_row_group[value] = [row]
10168 else:
10169 grouped_row_group[value].append(row)
10170 return grouped_row_group
10171
10172 - def as_list(self,
10173 compact=True,
10174 storage_to_dict=True,
10175 datetime_to_str=True,
10176 custom_types=None):
10177 """
10178 returns the data as a list or dictionary.
10179 :param storage_to_dict: when True returns a dict, otherwise a list(default True)
10180 :param datetime_to_str: convert datetime fields as strings (default True)
10181 """
10182 (oc, self.compact) = (self.compact, compact)
10183 if storage_to_dict:
10184 items = [item.as_dict(datetime_to_str, custom_types) for item in self]
10185 else:
10186 items = [item for item in self]
10187 self.compact = compact
10188 return items
10189
10190
10191 - def as_dict(self,
10192 key='id',
10193 compact=True,
10194 storage_to_dict=True,
10195 datetime_to_str=True,
10196 custom_types=None):
10197 """
10198 returns the data as a dictionary of dictionaries (storage_to_dict=True) or records (False)
10199
10200 :param key: the name of the field to be used as dict key, normally the id
10201 :param compact: ? (default True)
10202 :param storage_to_dict: when True returns a dict, otherwise a list(default True)
10203 :param datetime_to_str: convert datetime fields as strings (default True)
10204 """
10205
10206
10207 multi = False
10208 f = self.first()
10209 if f:
10210 multi = any([isinstance(v, f.__class__) for v in f.values()])
10211 if (not "." in key) and multi:
10212
10213 def new_key():
10214 i = 0
10215 while True:
10216 yield i
10217 i += 1
10218 key_generator = new_key()
10219 key = lambda r: key_generator.next()
10220
10221 rows = self.as_list(compact, storage_to_dict, datetime_to_str, custom_types)
10222 if isinstance(key,str) and key.count('.')==1:
10223 (table, field) = key.split('.')
10224 return dict([(r[table][field],r) for r in rows])
10225 elif isinstance(key,str):
10226 return dict([(r[key],r) for r in rows])
10227 else:
10228 return dict([(key(r),r) for r in rows])
10229
10231 """
10232 export data to csv, the first line contains the column names
10233
10234 :param ofile: where the csv must be exported to
10235 :param null: how null values must be represented (default '<NULL>')
10236 :param delimiter: delimiter to separate values (default ',')
10237 :param quotechar: character to use to quote string values (default '"')
10238 :param quoting: quote system, use csv.QUOTE_*** (default csv.QUOTE_MINIMAL)
10239 :param represent: use the fields .represent value (default False)
10240 :param colnames: list of column names to use (default self.colnames)
10241 This will only work when exporting rows objects!!!!
10242 DO NOT use this with db.export_to_csv()
10243 """
10244 delimiter = kwargs.get('delimiter', ',')
10245 quotechar = kwargs.get('quotechar', '"')
10246 quoting = kwargs.get('quoting', csv.QUOTE_MINIMAL)
10247 represent = kwargs.get('represent', False)
10248 writer = csv.writer(ofile, delimiter=delimiter,
10249 quotechar=quotechar, quoting=quoting)
10250 colnames = kwargs.get('colnames', self.colnames)
10251 write_colnames = kwargs.get('write_colnames',True)
10252
10253 if write_colnames:
10254 writer.writerow(colnames)
10255
10256 def none_exception(value):
10257 """
10258 returns a cleaned up value that can be used for csv export:
10259 - unicode text is encoded as such
10260 - None values are replaced with the given representation (default <NULL>)
10261 """
10262 if value is None:
10263 return null
10264 elif isinstance(value, unicode):
10265 return value.encode('utf8')
10266 elif isinstance(value,Reference):
10267 return int(value)
10268 elif hasattr(value, 'isoformat'):
10269 return value.isoformat()[:19].replace('T', ' ')
10270 elif isinstance(value, (list,tuple)):
10271 return bar_encode(value)
10272 return value
10273
10274 for record in self:
10275 row = []
10276 for col in colnames:
10277 if not REGEX_TABLE_DOT_FIELD.match(col):
10278 row.append(record._extra[col])
10279 else:
10280 (t, f) = col.split('.')
10281 field = self.db[t][f]
10282 if isinstance(record.get(t, None), (Row,dict)):
10283 value = record[t][f]
10284 else:
10285 value = record[f]
10286 if field.type=='blob' and not value is None:
10287 value = base64.b64encode(value)
10288 elif represent and field.represent:
10289 value = field.represent(value)
10290 row.append(none_exception(value))
10291 writer.writerow(row)
10292
10293 - def xml(self,strict=False,row_name='row',rows_name='rows'):
10294 """
10295 serializes the table using sqlhtml.SQLTABLE (if present)
10296 """
10297
10298 if strict:
10299 ncols = len(self.colnames)
10300 return '<%s>\n%s\n</%s>' % (rows_name,
10301 '\n'.join(row.as_xml(row_name=row_name,
10302 colnames=self.colnames) for
10303 row in self), rows_name)
10304
10305 import sqlhtml
10306 return sqlhtml.SQLTABLE(self).xml()
10307
10308 - def as_xml(self,row_name='row',rows_name='rows'):
10309 return self.xml(strict=True, row_name=row_name, rows_name=rows_name)
10310
10311 - def as_json(self, mode='object', default=None):
10329
10330
10331 as_csv = __str__
10332 json = as_json
10333
10334
10335
10336
10337
10338 -def test_all():
10339 """
10340
10341 >>> if len(sys.argv)<2: db = DAL(\"sqlite://test.db\")
10342 >>> if len(sys.argv)>1: db = DAL(sys.argv[1])
10343 >>> tmp = db.define_table('users',\
10344 Field('stringf', 'string', length=32, required=True),\
10345 Field('booleanf', 'boolean', default=False),\
10346 Field('passwordf', 'password', notnull=True),\
10347 Field('uploadf', 'upload'),\
10348 Field('blobf', 'blob'),\
10349 Field('integerf', 'integer', unique=True),\
10350 Field('doublef', 'double', unique=True,notnull=True),\
10351 Field('jsonf', 'json'),\
10352 Field('datef', 'date', default=datetime.date.today()),\
10353 Field('timef', 'time'),\
10354 Field('datetimef', 'datetime'),\
10355 migrate='test_user.table')
10356
10357 Insert a field
10358
10359 >>> db.users.insert(stringf='a', booleanf=True, passwordf='p', blobf='0A',\
10360 uploadf=None, integerf=5, doublef=3.14,\
10361 jsonf={"j": True},\
10362 datef=datetime.date(2001, 1, 1),\
10363 timef=datetime.time(12, 30, 15),\
10364 datetimef=datetime.datetime(2002, 2, 2, 12, 30, 15))
10365 1
10366
10367 Drop the table
10368
10369 >>> db.users.drop()
10370
10371 Examples of insert, select, update, delete
10372
10373 >>> tmp = db.define_table('person',\
10374 Field('name'),\
10375 Field('birth','date'),\
10376 migrate='test_person.table')
10377 >>> person_id = db.person.insert(name=\"Marco\",birth='2005-06-22')
10378 >>> person_id = db.person.insert(name=\"Massimo\",birth='1971-12-21')
10379
10380 commented len(db().select(db.person.ALL))
10381 commented 2
10382
10383 >>> me = db(db.person.id==person_id).select()[0] # test select
10384 >>> me.name
10385 'Massimo'
10386 >>> db.person[2].name
10387 'Massimo'
10388 >>> db.person(2).name
10389 'Massimo'
10390 >>> db.person(name='Massimo').name
10391 'Massimo'
10392 >>> db.person(db.person.name=='Massimo').name
10393 'Massimo'
10394 >>> row = db.person[2]
10395 >>> row.name == row['name'] == row['person.name'] == row('person.name')
10396 True
10397 >>> db(db.person.name=='Massimo').update(name='massimo') # test update
10398 1
10399 >>> db(db.person.name=='Marco').select().first().delete_record() # test delete
10400 1
10401
10402 Update a single record
10403
10404 >>> me.update_record(name=\"Max\")
10405 <Row {'name': 'Max', 'birth': datetime.date(1971, 12, 21), 'id': 2}>
10406 >>> me.name
10407 'Max'
10408
10409 Examples of complex search conditions
10410
10411 >>> len(db((db.person.name=='Max')&(db.person.birth<'2003-01-01')).select())
10412 1
10413 >>> len(db((db.person.name=='Max')&(db.person.birth<datetime.date(2003,01,01))).select())
10414 1
10415 >>> len(db((db.person.name=='Max')|(db.person.birth<'2003-01-01')).select())
10416 1
10417 >>> me = db(db.person.id==person_id).select(db.person.name)[0]
10418 >>> me.name
10419 'Max'
10420
10421 Examples of search conditions using extract from date/datetime/time
10422
10423 >>> len(db(db.person.birth.month()==12).select())
10424 1
10425 >>> len(db(db.person.birth.year()>1900).select())
10426 1
10427
10428 Example of usage of NULL
10429
10430 >>> len(db(db.person.birth==None).select()) ### test NULL
10431 0
10432 >>> len(db(db.person.birth!=None).select()) ### test NULL
10433 1
10434
10435 Examples of search conditions using lower, upper, and like
10436
10437 >>> len(db(db.person.name.upper()=='MAX').select())
10438 1
10439 >>> len(db(db.person.name.like('%ax')).select())
10440 1
10441 >>> len(db(db.person.name.upper().like('%AX')).select())
10442 1
10443 >>> len(db(~db.person.name.upper().like('%AX')).select())
10444 0
10445
10446 orderby, groupby and limitby
10447
10448 >>> people = db().select(db.person.name, orderby=db.person.name)
10449 >>> order = db.person.name|~db.person.birth
10450 >>> people = db().select(db.person.name, orderby=order)
10451
10452 >>> people = db().select(db.person.name, orderby=db.person.name, groupby=db.person.name)
10453
10454 >>> people = db().select(db.person.name, orderby=order, limitby=(0,100))
10455
10456 Example of one 2 many relation
10457
10458 >>> tmp = db.define_table('dog',\
10459 Field('name'),\
10460 Field('birth','date'),\
10461 Field('owner',db.person),\
10462 migrate='test_dog.table')
10463 >>> db.dog.insert(name='Snoopy', birth=None, owner=person_id)
10464 1
10465
10466 A simple JOIN
10467
10468 >>> len(db(db.dog.owner==db.person.id).select())
10469 1
10470
10471 >>> len(db().select(db.person.ALL, db.dog.name,left=db.dog.on(db.dog.owner==db.person.id)))
10472 1
10473
10474 Drop tables
10475
10476 >>> db.dog.drop()
10477 >>> db.person.drop()
10478
10479 Example of many 2 many relation and Set
10480
10481 >>> tmp = db.define_table('author', Field('name'),\
10482 migrate='test_author.table')
10483 >>> tmp = db.define_table('paper', Field('title'),\
10484 migrate='test_paper.table')
10485 >>> tmp = db.define_table('authorship',\
10486 Field('author_id', db.author),\
10487 Field('paper_id', db.paper),\
10488 migrate='test_authorship.table')
10489 >>> aid = db.author.insert(name='Massimo')
10490 >>> pid = db.paper.insert(title='QCD')
10491 >>> tmp = db.authorship.insert(author_id=aid, paper_id=pid)
10492
10493 Define a Set
10494
10495 >>> authored_papers = db((db.author.id==db.authorship.author_id)&(db.paper.id==db.authorship.paper_id))
10496 >>> rows = authored_papers.select(db.author.name, db.paper.title)
10497 >>> for row in rows: print row.author.name, row.paper.title
10498 Massimo QCD
10499
10500 Example of search condition using belongs
10501
10502 >>> set = (1, 2, 3)
10503 >>> rows = db(db.paper.id.belongs(set)).select(db.paper.ALL)
10504 >>> print rows[0].title
10505 QCD
10506
10507 Example of search condition using nested select
10508
10509 >>> nested_select = db()._select(db.authorship.paper_id)
10510 >>> rows = db(db.paper.id.belongs(nested_select)).select(db.paper.ALL)
10511 >>> print rows[0].title
10512 QCD
10513
10514 Example of expressions
10515
10516 >>> mynumber = db.define_table('mynumber', Field('x', 'integer'))
10517 >>> db(mynumber).delete()
10518 0
10519 >>> for i in range(10): tmp = mynumber.insert(x=i)
10520 >>> db(mynumber).select(mynumber.x.sum())[0](mynumber.x.sum())
10521 45
10522
10523 >>> db(mynumber.x+2==5).select(mynumber.x + 2)[0](mynumber.x + 2)
10524 5
10525
10526 Output in csv
10527
10528 >>> print str(authored_papers.select(db.author.name, db.paper.title)).strip()
10529 author.name,paper.title\r
10530 Massimo,QCD
10531
10532 Delete all leftover tables
10533
10534 >>> DAL.distributed_transaction_commit(db)
10535
10536 >>> db.mynumber.drop()
10537 >>> db.authorship.drop()
10538 >>> db.author.drop()
10539 >>> db.paper.drop()
10540 """
10541
10542
10543
10544
10545 SQLField = Field
10546 SQLTable = Table
10547 SQLXorable = Expression
10548 SQLQuery = Query
10549 SQLSet = Set
10550 SQLRows = Rows
10551 SQLStorage = Row
10552 SQLDB = DAL
10553 GQLDB = DAL
10554 DAL.Field = Field
10555 DAL.Table = Table
10556
10557
10558
10559
10560
10561 -def geoPoint(x,y):
10562 return "POINT (%f %f)" % (x,y)
10563
10565 return "LINESTRING (%s)" % ','.join("%f %f" % item for item in line)
10566
10568 return "POLYGON ((%s))" % ','.join("%f %f" % item for item in line)
10569
10570
10571
10572
10573
10574 if __name__ == '__main__':
10575 import doctest
10576 doctest.testmod()
10577